diff --git a/Cargo.lock b/Cargo.lock index d510da7..ca3b1f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1193,6 +1193,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/crates/rugpi-bakery/Cargo.toml b/crates/rugpi-bakery/Cargo.toml index 19b135e..634560e 100644 --- a/crates/rugpi-bakery/Cargo.toml +++ b/crates/rugpi-bakery/Cargo.toml @@ -19,5 +19,5 @@ sha1 = "0.10.5" tempfile = "3.8.1" thiserror = "1.0.43" toml = "0.8.8" -url = "2.4.0" +url = { version = "2.4.0", features = ["serde"] } xscript.workspace = true diff --git a/crates/rugpi-bakery/src/bake/customize.rs b/crates/rugpi-bakery/src/bake/customize.rs new file mode 100644 index 0000000..ee07011 --- /dev/null +++ b/crates/rugpi-bakery/src/bake/customize.rs @@ -0,0 +1,205 @@ +//! Applies a set of recipes to a system. + +use std::{ + collections::{HashMap, HashSet}, + fs, + ops::Deref, + path::Path, + sync::Arc, +}; + +use anyhow::{anyhow, bail}; +use clap::Parser; +use rugpi_common::{mount::Mounted, Anyhow}; +use tempfile::tempdir; +use xscript::{cmd, run, vars, ParentEnv, Run}; + +use crate::project::{ + config::Architecture, + layers::LayerConfig, + library::Library, + recipes::{Recipe, StepKind}, + Project, +}; + +/// The arguments of the `customize` command. +#[derive(Debug, Parser)] +pub struct CustomizeTask { + /// The source archive with the original system. + src: String, + /// The destination archive with the modified system. + dest: String, +} + +pub fn customize( + project: &Project, + arch: Architecture, + layer: &LayerConfig, + src: &Path, + target: &Path, +) -> Anyhow<()> { + let library = project.load_library()?; + // Collect the recipes to apply. + let jobs = recipe_schedule(layer, &library)?; + // Prepare system chroot. + let root_dir = tempdir()?; + let root_dir_path = root_dir.path(); + println!("Extracting system files..."); + run!(["tar", "-x", "-f", &src, "-C", root_dir_path])?; + apply_recipes(arch, &jobs, root_dir_path)?; + println!("Packing system files..."); + run!(["tar", "-c", "-f", &target, "-C", root_dir_path, "."])?; + Ok(()) +} + +struct RecipeJob { + recipe: Arc, + parameters: HashMap, +} + +fn recipe_schedule(layer: &LayerConfig, library: &Library) -> Anyhow> { + let mut stack = layer + .recipes + .iter() + .map(|name| { + library + .lookup(library.repositories.root_repository, name.deref()) + .ok_or_else(|| anyhow!("recipe with name {name} not found")) + }) + .collect::>>()?; + let mut enabled = stack.iter().cloned().collect::>(); + while let Some(idx) = stack.pop() { + let recipe = &library.recipes[idx]; + for name in &recipe.info.dependencies { + let dependency_idx = library + .lookup(recipe.repository, name.deref()) + .ok_or_else(|| anyhow!("recipe with name {name} not found"))?; + if enabled.insert(dependency_idx) { + stack.push(dependency_idx); + } + } + } + for excluded in &layer.exclude { + let excluded = library + .lookup(library.repositories.root_repository, excluded.deref()) + .ok_or_else(|| anyhow!("recipe with name {excluded} not found"))?; + enabled.remove(&excluded); + } + let parameters = layer + .parameters + .iter() + .map(|(name, parameters)| { + Ok(( + library + .lookup(library.repositories.root_repository, name.deref()) + .ok_or_else(|| anyhow!("recipe with name {name} not found"))?, + parameters, + )) + }) + .collect::>>()?; + let mut recipes = enabled + .into_iter() + .map(|idx| { + let recipe = library.recipes[idx].clone(); + let recipe_params = parameters.get(&idx); + if let Some(params) = recipe_params { + for param_name in params.keys() { + if !recipe.info.parameters.contains_key(param_name) { + bail!( + "unknown parameter `{param_name}` of recipe `{}`", + recipe.name + ); + } + } + } + let mut parameters = HashMap::new(); + for (name, def) in &recipe.info.parameters { + if let Some(params) = recipe_params { + if let Some(value) = params.get(name) { + parameters.insert(name.to_owned(), value.to_string()); + continue; + } + } + if let Some(default) = &def.default { + parameters.insert(name.to_owned(), default.to_string()); + continue; + } + bail!("unable to find value for parameter `{name}`"); + } + Ok(RecipeJob { recipe, parameters }) + }) + .collect::, _>>()?; + // 4️⃣ Sort recipes by priority. + recipes.sort_by_key(|job| -job.recipe.info.priority); + Ok(recipes) +} + +fn apply_recipes(arch: Architecture, jobs: &Vec, root_dir_path: &Path) -> Anyhow<()> { + let _mounted_dev = Mounted::bind("/dev", root_dir_path.join("dev"))?; + let _mounted_dev_pts = Mounted::bind("/dev/pts", root_dir_path.join("dev/pts"))?; + let _mounted_sys = Mounted::bind("/sys", root_dir_path.join("sys"))?; + let _mounted_proc = Mounted::mount_fs("proc", "proc", root_dir_path.join("proc"))?; + let _mounted_run = Mounted::mount_fs("tmpfs", "tmpfs", root_dir_path.join("run"))?; + let _mounted_tmp = Mounted::mount_fs("tmpfs", "tmpfs", root_dir_path.join("tmp"))?; + + let bakery_recipe_path = root_dir_path.join("run/rugpi/bakery/recipe"); + fs::create_dir_all(&bakery_recipe_path)?; + + for (idx, job) in jobs.iter().enumerate() { + let recipe = &job.recipe; + println!( + "[{:>2}/{}] {} {:?}", + idx + 1, + jobs.len(), + recipe + .info + .description + .as_deref() + .unwrap_or(recipe.name.deref()), + &job.parameters, + ); + let _mounted_recipe = Mounted::bind(&recipe.path, &bakery_recipe_path)?; + + for step in &recipe.steps { + println!(" - {}", step.filename); + match &step.kind { + StepKind::Packages { packages } => { + let mut cmd = cmd!("chroot", root_dir_path, "apt-get", "install", "-y"); + cmd.extend_args(packages); + ParentEnv.run(cmd.with_vars(vars! { + DEBIAN_FRONTEND = "noninteractive" + }))?; + } + StepKind::Install => { + let script = format!("/run/rugpi/bakery/recipe/steps/{}", step.filename); + let mut vars = vars! { + DEBIAN_FRONTEND = "noninteractive", + RUGPI_ROOT_DIR = "/", + RUGPI_ARCH = arch.as_str(), + RECIPE_DIR = "/run/rugpi/bakery/recipe/", + RECIPE_STEP_PATH = &script, + }; + for (name, value) in &job.parameters { + vars.set(format!("RECIPE_PARAM_{}", name.to_uppercase()), value); + } + run!(["chroot", root_dir_path, &script].with_vars(vars))?; + } + StepKind::Run => { + let script = recipe.path.join("steps").join(&step.filename); + let mut vars = vars! { + DEBIAN_FRONTEND = "noninteractive", + RUGPI_ROOT_DIR = root_dir_path, + RUGPI_ARCH = arch.as_str(), + RECIPE_DIR = &recipe.path, + RECIPE_STEP_PATH = &script, + }; + for (name, value) in &job.parameters { + vars.set(format!("RECIPE_PARAM_{}", name.to_uppercase()), value); + } + run!([&script].with_vars(vars))?; + } + } + } + } + Ok(()) +} diff --git a/crates/rugpi-bakery/src/tasks/bake.rs b/crates/rugpi-bakery/src/bake/image.rs similarity index 91% rename from crates/rugpi-bakery/src/tasks/bake.rs rename to crates/rugpi-bakery/src/bake/image.rs index 8351ece..68f5785 100644 --- a/crates/rugpi-bakery/src/tasks/bake.rs +++ b/crates/rugpi-bakery/src/bake/image.rs @@ -2,7 +2,6 @@ use std::{fs, path::Path}; -use clap::Parser; use rugpi_common::{ boot::{uboot::UBootEnv, BootFlow}, loop_dev::LoopDevice, @@ -15,21 +14,11 @@ use xscript::{run, Run}; use crate::project::{ config::{Architecture, IncludeFirmware}, - Project, + images::ImageConfig, }; -#[derive(Debug, Parser)] -pub struct BakeTask { - /// The archive with the system files. - archive: String, - /// The output image. - image: String, -} - -pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> { - let archive = Path::new(&task.archive); - let image = Path::new(&task.image); - let size = calculate_image_size(archive)?; +pub fn make_image(image_config: &ImageConfig, src: &Path, image: &Path) -> Anyhow<()> { + let size = calculate_image_size(src)?; println!("Size: {} bytes", size); fs::remove_file(image).ok(); println!("Creating image..."); @@ -56,19 +45,19 @@ pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> { let config_dir_path = config_dir.path(); let mounted_config = Mounted::mount(loop_device.partition(1), config_dir_path)?; let ctx = BakeCtx { - project, + config: image_config, mounted_boot, mounted_root, mounted_config, }; - run!(["tar", "-x", "-f", &task.archive, "-C", root_dir_path])?; + run!(["tar", "-x", "-f", src, "-C", root_dir_path])?; println!("Patching boot configuration..."); patch_boot(ctx.mounted_boot.path(), format!("PARTUUID={disk_id}-05"))?; println!("Patching `config.txt`..."); patch_config(boot_dir.join("config.txt"))?; - match project.config.boot_flow { + match image_config.boot_flow { BootFlow::Tryboot => setup_tryboot_boot_flow(&ctx)?, BootFlow::UBoot => setup_uboot_boot_flow(&ctx)?, } @@ -78,7 +67,7 @@ pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> { ctx.mounted_boot.path().join("second.scr"), )?; - match project.config.include_firmware { + match image_config.include_firmware { IncludeFirmware::None => { /* Do not include any firmware. */ } IncludeFirmware::Pi4 => include_pi4_firmware(ctx.mounted_config.path())?, IncludeFirmware::Pi5 => include_pi5_firmware(ctx.mounted_config.path())?, @@ -91,7 +80,7 @@ pub fn run(project: &Project, task: &BakeTask) -> Anyhow<()> { } struct BakeCtx<'p> { - project: &'p Project, + config: &'p ImageConfig, mounted_boot: Mounted, #[allow(unused)] mounted_root: Mounted, @@ -124,7 +113,7 @@ fn setup_uboot_boot_flow(ctx: &BakeCtx) -> Anyhow<()> { ctx.mounted_config.path() ])?; std::fs::remove_file(ctx.mounted_config.path().join("kernel8.img"))?; - match ctx.project.config.architecture { + match ctx.config.architecture { Architecture::Arm64 => { std::fs::copy( "/usr/share/rugpi/boot/u-boot/arm64_config.txt", diff --git a/crates/rugpi-bakery/src/bake/mod.rs b/crates/rugpi-bakery/src/bake/mod.rs new file mode 100644 index 0000000..4096f26 --- /dev/null +++ b/crates/rugpi-bakery/src/bake/mod.rs @@ -0,0 +1,88 @@ +//! Functionality for baking layers and images. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, bail}; +use rugpi_common::{loop_dev::LoopDevice, mount::Mounted, Anyhow}; +use tempfile::tempdir; +use xscript::{run, Run}; + +use crate::{ + project::{config::Architecture, Project}, + utils::{download, sha1}, +}; + +pub mod customize; +pub mod image; + +pub fn bake_image(project: &Project, image: &str, output: &Path) -> Anyhow<()> { + let image_config = project + .config + .images + .get(image) + .ok_or_else(|| anyhow!("unable to find image {image}"))?; + let baked_layer = bake_layer(project, image_config.architecture, &image_config.layer)?; + image::make_image(image_config, &baked_layer, output) +} + +pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow { + let layer_config = project + .config + .layers + .get(layer) + .ok_or_else(|| anyhow!("unable to find layer {layer}"))?; + if let Some(url) = &layer_config.url { + let layer_id = sha1(url); + let system_tar = project + .dir + .join(format!(".rugpi/layers/{layer_id}/system.tar")); + if !system_tar.exists() { + extract(url, &system_tar)?; + } + Ok(system_tar) + } else if let Some(parent) = &layer_config.parent { + let src = bake_layer(project, arch, parent)?; + let mut layer_string = layer.to_owned(); + layer_string.push('.'); + layer_string.push_str(arch.as_str()); + let layer_id = sha1(&layer_string); + let target = project + .dir + .join(format!(".rugpi/layers/{layer_id}/system.tar")); + fs::create_dir_all(target.parent().unwrap()).ok(); + if !target.exists() { + customize::customize(project, arch, layer_config, &src, &target)?; + } + Ok(target) + } else { + bail!("invalid layer configuration") + } +} + +fn extract(image_url: &str, layer_path: &Path) -> Anyhow<()> { + let mut image_path = download(image_url)?; + if image_path.extension() == Some("xz".as_ref()) { + let decompressed_image_path = image_path.with_extension(""); + if !decompressed_image_path.is_file() { + eprintln!("Decompressing XZ image..."); + run!(["xz", "-d", "-k", image_path])?; + } + image_path = decompressed_image_path; + } + eprintln!("Creating `.tar` archive with system files..."); + if let Some(parent) = layer_path.parent() { + if !parent.exists() { + fs::create_dir_all(parent)?; + } + } + let loop_dev = LoopDevice::attach(image_path)?; + let temp_dir = tempdir()?; + let temp_dir_path = temp_dir.path(); + let _mounted_root = Mounted::mount(loop_dev.partition(2), temp_dir_path)?; + let _mounted_boot = Mounted::mount(loop_dev.partition(1), temp_dir_path.join("boot"))?; + run!(["tar", "-c", "-f", &layer_path, "-C", temp_dir_path, "."])?; + Ok(()) +} diff --git a/crates/rugpi-bakery/src/main.rs b/crates/rugpi-bakery/src/main.rs index 9b819e7..f37ddb1 100644 --- a/crates/rugpi-bakery/src/main.rs +++ b/crates/rugpi-bakery/src/main.rs @@ -5,14 +5,10 @@ use std::{ use clap::Parser; use colored::Colorize; -use project::{repositories::Source, ProjectLoader}; +use project::{config::Architecture, repositories::Source, ProjectLoader}; use rugpi_common::Anyhow; -use tasks::{ - bake::{self, BakeTask}, - customize::{self, CustomizeTask}, - extract::ExtractTask, -}; +pub mod bake; pub mod idx_vec; pub mod project; pub mod tasks; @@ -28,14 +24,28 @@ pub struct Args { task: Task, } +#[derive(Debug, Parser)] +pub enum BakeCommand { + Image { + image: String, + output: PathBuf, + }, + Layer { + #[clap(long)] + arch: Architecture, + layer: String, + }, +} + #[derive(Debug, Parser)] pub enum Task { - /// Extract all system files from a given base image. - Extract(ExtractTask), - /// Apply modification to the system. - Customize(CustomizeTask), + // /// Extract all system files from a given base image. + // Extract(ExtractTask), + // /// Apply modification to the system. + // Customize(CustomizeTask), /// Bake a final image for distribution. - Bake(BakeTask), + #[clap(subcommand)] + Bake(BakeCommand), /// Spawn a shell in the Rugpi Bakery Docker container. Shell, Update(UpdateTask), @@ -54,15 +64,14 @@ fn main() -> Anyhow<()> { .with_config_file(args.config.as_deref()) .load()?; match &args.task { - Task::Extract(task) => { - task.run()?; - } - Task::Customize(task) => { - customize::run(&project, task)?; - } - Task::Bake(task) => { - bake::run(&project, task)?; - } + Task::Bake(task) => match task { + BakeCommand::Image { image, output } => { + bake::bake_image(&project, image, output)?; + } + BakeCommand::Layer { layer, arch } => { + bake::bake_layer(&project, *arch, layer)?; + } + }, Task::Shell => { let zsh_prog = CString::new("/bin/zsh")?; nix::unistd::execv::<&CStr>(&zsh_prog, &[])?; diff --git a/crates/rugpi-bakery/src/project/config.rs b/crates/rugpi-bakery/src/project/config.rs index 01c9375..c4b75f8 100644 --- a/crates/rugpi-bakery/src/project/config.rs +++ b/crates/rugpi-bakery/src/project/config.rs @@ -1,19 +1,13 @@ //! Project configuration. -use std::{ - collections::{HashMap, HashSet}, - fs, - path::Path, -}; +use std::{collections::HashMap, fs, path::Path}; use anyhow::Context; -use rugpi_common::{boot::BootFlow, Anyhow}; +use clap::ValueEnum; +use rugpi_common::Anyhow; use serde::{Deserialize, Serialize}; -use super::{ - recipes::{ParameterValue, RecipeName}, - repositories::Source, -}; +use super::{images::ImageConfig, layers::LayerConfig, repositories::Source}; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -21,24 +15,12 @@ pub struct BakeryConfig { /// The repositories to use. #[serde(default)] pub repositories: HashMap, - /// The recipes to include. + /// The layers of the project. #[serde(default)] - pub recipes: HashSet, - /// The recipes to exclude. + pub layers: HashMap, + /// The images of the project. #[serde(default)] - pub exclude: HashSet, - /// Parameters for the recipes. - #[serde(default)] - pub parameters: HashMap>, - /// Indicates whether to include firmware files in the image. - #[serde(default)] - pub include_firmware: IncludeFirmware, - /// The target architecture to build an image for. - #[serde(default)] - pub architecture: Architecture, - /// Indicates which boot flow to use for the image. - #[serde(default)] - pub boot_flow: BootFlow, + pub images: HashMap, } impl BakeryConfig { @@ -61,7 +43,8 @@ pub enum IncludeFirmware { Pi5, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, ValueEnum)] +#[clap(rename_all = "lowercase")] #[serde(rename_all = "lowercase")] pub enum Architecture { #[default] diff --git a/crates/rugpi-bakery/src/project/images.rs b/crates/rugpi-bakery/src/project/images.rs new file mode 100644 index 0000000..e58c866 --- /dev/null +++ b/crates/rugpi-bakery/src/project/images.rs @@ -0,0 +1,20 @@ +use rugpi_common::boot::BootFlow; +use serde::{Deserialize, Serialize}; + +use super::config::{Architecture, IncludeFirmware}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ImageConfig { + /// The layer to use for the image. + pub layer: String, + /// Indicates whether to include firmware files in the image. + #[serde(default)] + pub include_firmware: IncludeFirmware, + /// The target architecture to build an image for. + #[serde(default)] + pub architecture: Architecture, + /// Indicates which boot flow to use for the image. + #[serde(default)] + pub boot_flow: BootFlow, +} diff --git a/crates/rugpi-bakery/src/project/layers.rs b/crates/rugpi-bakery/src/project/layers.rs new file mode 100644 index 0000000..ace468e --- /dev/null +++ b/crates/rugpi-bakery/src/project/layers.rs @@ -0,0 +1,22 @@ +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +use super::recipes::{ParameterValue, RecipeName}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LayerConfig { + /// An URL to fetch the layer from. + pub url: Option, + pub parent: Option, + /// The recipes to include. + #[serde(default)] + pub recipes: HashSet, + /// The recipes to exclude. + #[serde(default)] + pub exclude: HashSet, + /// Parameters for the recipes. + #[serde(default)] + pub parameters: HashMap>, +} diff --git a/crates/rugpi-bakery/src/project/mod.rs b/crates/rugpi-bakery/src/project/mod.rs index 788ddbb..deb0fb5 100644 --- a/crates/rugpi-bakery/src/project/mod.rs +++ b/crates/rugpi-bakery/src/project/mod.rs @@ -7,6 +7,8 @@ use rugpi_common::Anyhow; use self::{config::BakeryConfig, library::Library, repositories::ProjectRepositories}; pub mod config; +pub mod images; +pub mod layers; pub mod library; pub mod recipes; pub mod repositories; diff --git a/crates/rugpi-bakery/src/tasks.rs b/crates/rugpi-bakery/src/tasks.rs index b2f7d25..d0fdf69 100644 --- a/crates/rugpi-bakery/src/tasks.rs +++ b/crates/rugpi-bakery/src/tasks.rs @@ -1,3 +1,3 @@ -pub mod bake; -pub mod customize; -pub mod extract; +// pub mod bake; +// pub mod customize; +// pub mod extract; diff --git a/crates/rugpi-bakery/src/utils.rs b/crates/rugpi-bakery/src/utils.rs index 4bf9cb4..5098f1f 100644 --- a/crates/rugpi-bakery/src/utils.rs +++ b/crates/rugpi-bakery/src/utils.rs @@ -28,3 +28,9 @@ pub fn download(url: &str) -> Anyhow { } Ok(cache_file_path) } + +pub fn sha1(string: &str) -> String { + let mut hasher = Sha1::new(); + hasher.update(string.as_bytes()); + hex::encode(hasher.finalize()) +} diff --git a/template b/template index b413cc2..364adb4 160000 --- a/template +++ b/template @@ -1 +1 @@ -Subproject commit b413cc2cfcd0dcdec69eb9263916c661be3e3cac +Subproject commit 364adb469124d8aed9ed4532c45f9ca56294d01f