chore: prototypical layer implementation

crates/rugpi-bakery/Cargo.toml

@@ -19,5 +19,5 @@ sha1 = "0.10.5"
 tempfile = "3.8.1"
 thiserror = "1.0.43"
 toml = "0.8.8"
-url = "2.4.0"
+url = { version = "2.4.0", features = ["serde"] }
 xscript.workspace = true

crates/rugpi-bakery/src/bake/customize.rs

@@ -0,0 +1,205 @@
+//! Applies a set of recipes to a system.
+
+use std::{
+    collections::{HashMap, HashSet},
+    fs,
+    ops::Deref,
+    path::Path,
+    sync::Arc,
+};
+
+use anyhow::{anyhow, bail};
+use clap::Parser;
+use rugpi_common::{mount::Mounted, Anyhow};
+use tempfile::tempdir;
+use xscript::{cmd, run, vars, ParentEnv, Run};
+
+use crate::project::{
+    config::Architecture,
+    layers::LayerConfig,
+    library::Library,
+    recipes::{Recipe, StepKind},
+    Project,
+};
+
+/// The arguments of the `customize` command.
+#[derive(Debug, Parser)]
+pub struct CustomizeTask {
+    /// The source archive with the original system.
+    src: String,
+    /// The destination archive with the modified system.
+    dest: String,
+}
+
+pub fn customize(
+    project: &Project,
+    arch: Architecture,
+    layer: &LayerConfig,
+    src: &Path,
+    target: &Path,
+) -> Anyhow<()> {
+    let library = project.load_library()?;
+    // Collect the recipes to apply.
+    let jobs = recipe_schedule(layer, &library)?;
+    // Prepare system chroot.
+    let root_dir = tempdir()?;
+    let root_dir_path = root_dir.path();
+    println!("Extracting system files...");
+    run!(["tar", "-x", "-f", &src, "-C", root_dir_path])?;
+    apply_recipes(arch, &jobs, root_dir_path)?;
+    println!("Packing system files...");
+    run!(["tar", "-c", "-f", &target, "-C", root_dir_path, "."])?;
+    Ok(())
+}
+
+struct RecipeJob {
+    recipe: Arc<Recipe>,
+    parameters: HashMap<String, String>,
+}
+
+fn recipe_schedule(layer: &LayerConfig, library: &Library) -> Anyhow<Vec<RecipeJob>> {
+    let mut stack = layer
+        .recipes
+        .iter()
+        .map(|name| {
+            library
+                .lookup(library.repositories.root_repository, name.deref())
+                .ok_or_else(|| anyhow!("recipe with name {name} not found"))
+        })
+        .collect::<Anyhow<Vec<_>>>()?;
+    let mut enabled = stack.iter().cloned().collect::<HashSet<_>>();
+    while let Some(idx) = stack.pop() {
+        let recipe = &library.recipes[idx];
+        for name in &recipe.info.dependencies {
+            let dependency_idx = library
+                .lookup(recipe.repository, name.deref())
+                .ok_or_else(|| anyhow!("recipe with name {name} not found"))?;
+            if enabled.insert(dependency_idx) {
+                stack.push(dependency_idx);
+            }
+        }
+    }
+    for excluded in &layer.exclude {
+        let excluded = library
+            .lookup(library.repositories.root_repository, excluded.deref())
+            .ok_or_else(|| anyhow!("recipe with name {excluded} not found"))?;
+        enabled.remove(&excluded);
+    }
+    let parameters = layer
+        .parameters
+        .iter()
+        .map(|(name, parameters)| {
+            Ok((
+                library
+                    .lookup(library.repositories.root_repository, name.deref())
+                    .ok_or_else(|| anyhow!("recipe with name {name} not found"))?,
+                parameters,
+            ))
+        })
+        .collect::<Anyhow<HashMap<_, _>>>()?;
+    let mut recipes = enabled
+        .into_iter()
+        .map(|idx| {
+            let recipe = library.recipes[idx].clone();
+            let recipe_params = parameters.get(&idx);
+            if let Some(params) = recipe_params {
+                for param_name in params.keys() {
+                    if !recipe.info.parameters.contains_key(param_name) {
+                        bail!(
+                            "unknown parameter `{param_name}` of recipe `{}`",
+                            recipe.name
+                        );
+                    }
+                }
+            }
+            let mut parameters = HashMap::new();
+            for (name, def) in &recipe.info.parameters {
+                if let Some(params) = recipe_params {
+                    if let Some(value) = params.get(name) {
+                        parameters.insert(name.to_owned(), value.to_string());
+                        continue;
+                    }
+                }
+                if let Some(default) = &def.default {
+                    parameters.insert(name.to_owned(), default.to_string());
+                    continue;
+                }
+                bail!("unable to find value for parameter `{name}`");
+            }
+            Ok(RecipeJob { recipe, parameters })
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+    // 4️⃣ Sort recipes by priority.
+    recipes.sort_by_key(|job| -job.recipe.info.priority);
+    Ok(recipes)
+}
+
+fn apply_recipes(arch: Architecture, jobs: &Vec<RecipeJob>, root_dir_path: &Path) -> Anyhow<()> {
+    let _mounted_dev = Mounted::bind("/dev", root_dir_path.join("dev"))?;
+    let _mounted_dev_pts = Mounted::bind("/dev/pts", root_dir_path.join("dev/pts"))?;
+    let _mounted_sys = Mounted::bind("/sys", root_dir_path.join("sys"))?;
+    let _mounted_proc = Mounted::mount_fs("proc", "proc", root_dir_path.join("proc"))?;
+    let _mounted_run = Mounted::mount_fs("tmpfs", "tmpfs", root_dir_path.join("run"))?;
+    let _mounted_tmp = Mounted::mount_fs("tmpfs", "tmpfs", root_dir_path.join("tmp"))?;
+
+    let bakery_recipe_path = root_dir_path.join("run/rugpi/bakery/recipe");
+    fs::create_dir_all(&bakery_recipe_path)?;
+
+    for (idx, job) in jobs.iter().enumerate() {
+        let recipe = &job.recipe;
+        println!(
+            "[{:>2}/{}] {} {:?}",
+            idx + 1,
+            jobs.len(),
+            recipe
+                .info
+                .description
+                .as_deref()
+                .unwrap_or(recipe.name.deref()),
+            &job.parameters,
+        );
+        let _mounted_recipe = Mounted::bind(&recipe.path, &bakery_recipe_path)?;
+
+        for step in &recipe.steps {
+            println!("    - {}", step.filename);
+            match &step.kind {
+                StepKind::Packages { packages } => {
+                    let mut cmd = cmd!("chroot", root_dir_path, "apt-get", "install", "-y");
+                    cmd.extend_args(packages);
+                    ParentEnv.run(cmd.with_vars(vars! {
+                        DEBIAN_FRONTEND = "noninteractive"
+                    }))?;
+                }
+                StepKind::Install => {
+                    let script = format!("/run/rugpi/bakery/recipe/steps/{}", step.filename);
+                    let mut vars = vars! {
+                        DEBIAN_FRONTEND = "noninteractive",
+                        RUGPI_ROOT_DIR = "/",
+                        RUGPI_ARCH = arch.as_str(),
+                        RECIPE_DIR = "/run/rugpi/bakery/recipe/",
+                        RECIPE_STEP_PATH = &script,
+                    };
+                    for (name, value) in &job.parameters {
+                        vars.set(format!("RECIPE_PARAM_{}", name.to_uppercase()), value);
+                    }
+                    run!(["chroot", root_dir_path, &script].with_vars(vars))?;
+                }
+                StepKind::Run => {
+                    let script = recipe.path.join("steps").join(&step.filename);
+                    let mut vars = vars! {
+                        DEBIAN_FRONTEND = "noninteractive",
+                        RUGPI_ROOT_DIR = root_dir_path,
+                        RUGPI_ARCH = arch.as_str(),
+                        RECIPE_DIR = &recipe.path,
+                        RECIPE_STEP_PATH = &script,
+                    };
+                    for (name, value) in &job.parameters {
+                        vars.set(format!("RECIPE_PARAM_{}", name.to_uppercase()), value);
+                    }
+                    run!([&script].with_vars(vars))?;
+                }
+            }
+        }
+    }
+    Ok(())
+}

crates/rugpi-bakery/src/tasks/bake.rs → crates/rugpi-bakery/src/bake/image.rs

@@ -2,7 +2,6 @@
 
 use std::{fs, path::Path};
 
-use clap::Parser;
 use rugpi_common::{
     boot::{uboot::UBootEnv, BootFlow},
     loop_dev::LoopDevice,
@@ -15,21 +14,11 @@ use xscript::{run, Run};
 
 use crate::project::{
     config::{Architecture, IncludeFirmware},
-    Project,
+    images::ImageConfig,
 };
 
-#[derive(Debug, Parser)]
-pub struct BakeTask {
-    /// The archive with the system files.
-    archive: String,
-    /// The output image.
-    image: String,
-}
-
-pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> {
-    let archive = Path::new(&task.archive);
-    let image = Path::new(&task.image);
-    let size = calculate_image_size(archive)?;
+pub fn make_image(image_config: &ImageConfig, src: &Path, image: &Path) -> Anyhow<()> {
+    let size = calculate_image_size(src)?;
     println!("Size: {} bytes", size);
     fs::remove_file(image).ok();
     println!("Creating image...");
@@ -56,19 +45,19 @@ pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> {
         let config_dir_path = config_dir.path();
         let mounted_config = Mounted::mount(loop_device.partition(1), config_dir_path)?;
         let ctx = BakeCtx {
-            project,
+            config: image_config,
             mounted_boot,
             mounted_root,
             mounted_config,
         };
 
-        run!(["tar", "-x", "-f", &task.archive, "-C", root_dir_path])?;
+        run!(["tar", "-x", "-f", src, "-C", root_dir_path])?;
         println!("Patching boot configuration...");
         patch_boot(ctx.mounted_boot.path(), format!("PARTUUID={disk_id}-05"))?;
         println!("Patching `config.txt`...");
         patch_config(boot_dir.join("config.txt"))?;
 
-        match project.config.boot_flow {
+        match image_config.boot_flow {
             BootFlow::Tryboot => setup_tryboot_boot_flow(&ctx)?,
             BootFlow::UBoot => setup_uboot_boot_flow(&ctx)?,
         }
@@ -78,7 +67,7 @@ pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> {
             ctx.mounted_boot.path().join("second.scr"),
         )?;
 
-        match project.config.include_firmware {
+        match image_config.include_firmware {
             IncludeFirmware::None => { /* Do not include any firmware. */ }
             IncludeFirmware::Pi4 => include_pi4_firmware(ctx.mounted_config.path())?,
             IncludeFirmware::Pi5 => include_pi5_firmware(ctx.mounted_config.path())?,
@@ -91,7 +80,7 @@ pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> {
 }
 
 struct BakeCtx<'p> {
-    project: &'p Project,
+    config: &'p ImageConfig,
     mounted_boot: Mounted,
     #[allow(unused)]
     mounted_root: Mounted,
@@ -124,7 +113,7 @@ fn setup_uboot_boot_flow(ctx: &BakeCtx) -> Anyhow<()> {
         ctx.mounted_config.path()
     ])?;
     std::fs::remove_file(ctx.mounted_config.path().join("kernel8.img"))?;
-    match ctx.project.config.architecture {
+    match ctx.config.architecture {
         Architecture::Arm64 => {
             std::fs::copy(
                 "/usr/share/rugpi/boot/u-boot/arm64_config.txt",

crates/rugpi-bakery/src/bake/mod.rs

@@ -0,0 +1,88 @@
+//! Functionality for baking layers and images.
+
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
+
+use anyhow::{anyhow, bail};
+use rugpi_common::{loop_dev::LoopDevice, mount::Mounted, Anyhow};
+use tempfile::tempdir;
+use xscript::{run, Run};
+
+use crate::{
+    project::{config::Architecture, Project},
+    utils::{download, sha1},
+};
+
+pub mod customize;
+pub mod image;
+
+pub fn bake_image(project: &Project, image: &str, output: &Path) -> Anyhow<()> {
+    let image_config = project
+        .config
+        .images
+        .get(image)
+        .ok_or_else(|| anyhow!("unable to find image {image}"))?;
+    let baked_layer = bake_layer(project, image_config.architecture, &image_config.layer)?;
+    image::make_image(image_config, &baked_layer, output)
+}
+
+pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow<PathBuf> {
+    let layer_config = project
+        .config
+        .layers
+        .get(layer)
+        .ok_or_else(|| anyhow!("unable to find layer {layer}"))?;
+    if let Some(url) = &layer_config.url {
+        let layer_id = sha1(url);
+        let system_tar = project
+            .dir
+            .join(format!(".rugpi/layers/{layer_id}/system.tar"));
+        if !system_tar.exists() {
+            extract(url, &system_tar)?;
+        }
+        Ok(system_tar)
+    } else if let Some(parent) = &layer_config.parent {
+        let src = bake_layer(project, arch, parent)?;
+        let mut layer_string = layer.to_owned();
+        layer_string.push('.');
+        layer_string.push_str(arch.as_str());
+        let layer_id = sha1(&layer_string);
+        let target = project
+            .dir
+            .join(format!(".rugpi/layers/{layer_id}/system.tar"));
+        fs::create_dir_all(target.parent().unwrap()).ok();
+        if !target.exists() {
+            customize::customize(project, arch, layer_config, &src, &target)?;
+        }
+        Ok(target)
+    } else {
+        bail!("invalid layer configuration")
+    }
+}
+
+fn extract(image_url: &str, layer_path: &Path) -> Anyhow<()> {
+    let mut image_path = download(image_url)?;
+    if image_path.extension() == Some("xz".as_ref()) {
+        let decompressed_image_path = image_path.with_extension("");
+        if !decompressed_image_path.is_file() {
+            eprintln!("Decompressing XZ image...");
+            run!(["xz", "-d", "-k", image_path])?;
+        }
+        image_path = decompressed_image_path;
+    }
+    eprintln!("Creating `.tar` archive with system files...");
+    if let Some(parent) = layer_path.parent() {
+        if !parent.exists() {
+            fs::create_dir_all(parent)?;
+        }
+    }
+    let loop_dev = LoopDevice::attach(image_path)?;
+    let temp_dir = tempdir()?;
+    let temp_dir_path = temp_dir.path();
+    let _mounted_root = Mounted::mount(loop_dev.partition(2), temp_dir_path)?;
+    let _mounted_boot = Mounted::mount(loop_dev.partition(1), temp_dir_path.join("boot"))?;
+    run!(["tar", "-c", "-f", &layer_path, "-C", temp_dir_path, "."])?;
+    Ok(())
+}