diff --git a/crates/rugpi-bakery/src/bake/customize.rs b/crates/rugpi-bakery/src/bake/customize.rs index ee07011..0631d58 100644 --- a/crates/rugpi-bakery/src/bake/customize.rs +++ b/crates/rugpi-bakery/src/bake/customize.rs @@ -14,12 +14,15 @@ use rugpi_common::{mount::Mounted, Anyhow}; use tempfile::tempdir; use xscript::{cmd, run, vars, ParentEnv, Run}; -use crate::project::{ - config::Architecture, - layers::LayerConfig, - library::Library, - recipes::{Recipe, StepKind}, - Project, +use crate::{ + caching::mtime, + project::{ + config::Architecture, + layers::{Layer, LayerConfig}, + library::Library, + recipes::{Recipe, StepKind}, + Project, + }, }; /// The arguments of the `customize` command. @@ -34,13 +37,23 @@ pub struct CustomizeTask { pub fn customize( project: &Project, arch: Architecture, - layer: &LayerConfig, + layer: &Layer, src: &Path, target: &Path, ) -> Anyhow<()> { let library = project.load_library()?; // Collect the recipes to apply. - let jobs = recipe_schedule(layer, &library)?; + let config = layer.config(arch).unwrap(); + let jobs = recipe_schedule(config, &library)?; + let last_modified = jobs + .iter() + .map(|job| job.recipe.modified) + .max() + .unwrap() + .max(layer.modified); + if target.exists() && last_modified < mtime(target)? { + return Ok(()); + } // Prepare system chroot. let root_dir = tempdir()?; let root_dir_path = root_dir.path(); diff --git a/crates/rugpi-bakery/src/bake/mod.rs b/crates/rugpi-bakery/src/bake/mod.rs index b9c7278..e15d08e 100644 --- a/crates/rugpi-bakery/src/bake/mod.rs +++ b/crates/rugpi-bakery/src/bake/mod.rs @@ -11,8 +11,8 @@ use tempfile::tempdir; use xscript::{run, Run}; use crate::{ + caching::{download, sha1}, project::{config::Architecture, Project}, - utils::{download, sha1}, }; pub mod customize; @@ -28,13 +28,12 @@ pub fn bake_image(project: &Project, image: &str, output: &Path) -> Anyhow<()> { image::make_image(image_config, &baked_layer, output) } -pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow { +pub fn bake_layer(project: &Project, arch: Architecture, layer_name: &str) -> Anyhow { let library = project.load_library()?; - let layer_config = &library.layers[library - .lookup_layer(library.repositories.root_repository, layer) - .unwrap()] - .config(arch) - .unwrap(); + let layer = &library.layers[library + .lookup_layer(library.repositories.root_repository, layer_name) + .unwrap()]; + let layer_config = layer.config(arch).unwrap(); if let Some(url) = &layer_config.url { let layer_id = sha1(url); let system_tar = project @@ -46,7 +45,7 @@ pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow< Ok(system_tar) } else if let Some(parent) = &layer_config.parent { let src = bake_layer(project, arch, parent)?; - let mut layer_string = layer.to_owned(); + let mut layer_string = layer_name.to_owned(); layer_string.push('.'); layer_string.push_str(arch.as_str()); let layer_id = sha1(&layer_string); @@ -54,9 +53,7 @@ pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow< .dir .join(format!(".rugpi/layers/{layer_id}/system.tar")); fs::create_dir_all(target.parent().unwrap()).ok(); - if !target.exists() { - customize::customize(project, arch, layer_config, &src, &target)?; - } + customize::customize(project, arch, layer, &src, &target)?; Ok(target) } else { bail!("invalid layer configuration") diff --git a/crates/rugpi-bakery/src/caching.rs b/crates/rugpi-bakery/src/caching.rs new file mode 100644 index 0000000..03f9da6 --- /dev/null +++ b/crates/rugpi-bakery/src/caching.rs @@ -0,0 +1,80 @@ +//! Utilities for caching. + +use std::{ + fs, io, + path::{Path, PathBuf}, + time::SystemTime, +}; + +use rugpi_common::Anyhow; +use serde::{Deserialize, Serialize}; +use sha1::{Digest, Sha1}; +use url::Url; +use xscript::{run, Run}; + +pub fn download(url: &str) -> Anyhow { + let url = url.parse::()?; + let Some(file_name) = url.path_segments().and_then(|segments| segments.last()) else { + anyhow::bail!("unable to obtain file name from URL"); + }; + let file_extension = file_name.split_once('.').map(|(_, extension)| extension); + let mut url_hasher = Sha1::new(); + url_hasher.update(url.as_str().as_bytes()); + let url_hash = url_hasher.finalize(); + let mut cache_file_name = hex::encode(url_hash); + if let Some(extension) = file_extension { + cache_file_name.push('.'); + cache_file_name.push_str(extension); + } + let cache_file_path = Path::new(".rugpi/cache").join(cache_file_name); + if !cache_file_path.exists() { + std::fs::create_dir_all(".rugpi/cache")?; + run!(["wget", "-O", &cache_file_path, url.as_str()])?; + } + Ok(cache_file_path) +} + +pub fn sha1(string: &str) -> String { + let mut hasher = Sha1::new(); + hasher.update(string.as_bytes()); + hex::encode(hasher.finalize()) +} + +/// Modification time in seconds since the UNIX epoch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ModificationTime(u64); + +impl ModificationTime { + /// Extract the modification time from the provided filesystem metadata. + fn from_metadata(metadata: &fs::Metadata) -> Result { + metadata.modified().map(|modified| { + ModificationTime( + modified + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + ) + }) + } +} + +/// The modification time of the given path. +pub fn mtime(path: &Path) -> Result { + fs::metadata(path).and_then(|metadata| ModificationTime::from_metadata(&metadata)) +} + +/// Recursively scans a path and return the latest modification time. +pub fn mtime_recursive(path: &Path) -> Result { + let mut time = mtime(path)?; + if path.is_dir() { + for entry in fs::read_dir(path)? { + let entry = entry?; + time = time.max(if entry.file_type()?.is_dir() { + mtime_recursive(&entry.path())? + } else { + ModificationTime::from_metadata(&entry.metadata()?)? + }); + } + } + Ok(time) +} diff --git a/crates/rugpi-bakery/src/main.rs b/crates/rugpi-bakery/src/main.rs index f79dbfe..19a828e 100644 --- a/crates/rugpi-bakery/src/main.rs +++ b/crates/rugpi-bakery/src/main.rs @@ -9,9 +9,9 @@ use project::{config::Architecture, repositories::Source, ProjectLoader}; use rugpi_common::Anyhow; pub mod bake; +pub mod caching; pub mod idx_vec; pub mod project; -pub mod utils; #[derive(Debug, Parser)] pub struct Args { diff --git a/crates/rugpi-bakery/src/project/layers.rs b/crates/rugpi-bakery/src/project/layers.rs index 61862a4..c65f43c 100644 --- a/crates/rugpi-bakery/src/project/layers.rs +++ b/crates/rugpi-bakery/src/project/layers.rs @@ -12,6 +12,7 @@ use super::{ config::Architecture, recipes::{ParameterValue, RecipeName}, }; +use crate::caching::ModificationTime; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -42,13 +43,22 @@ impl LayerConfig { } } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Layer { + pub modified: ModificationTime, pub default_config: Option, pub arch_configs: HashMap, } impl Layer { + pub fn new(modified: ModificationTime) -> Self { + Self { + modified, + default_config: None, + arch_configs: HashMap::new(), + } + } + /// The layer configuration for the given architecture. pub fn config(&self, arch: Architecture) -> Option<&LayerConfig> { self.arch_configs diff --git a/crates/rugpi-bakery/src/project/library.rs b/crates/rugpi-bakery/src/project/library.rs index 23adb0a..8d1dd80 100644 --- a/crates/rugpi-bakery/src/project/library.rs +++ b/crates/rugpi-bakery/src/project/library.rs @@ -8,7 +8,10 @@ use super::{ recipes::{Recipe, RecipeLoader}, repositories::{ProjectRepositories, RepositoryIdx}, }; -use crate::idx_vec::{new_idx_type, IdxVec}; +use crate::{ + caching::mtime, + idx_vec::{new_idx_type, IdxVec}, +}; pub struct Library { pub repositories: ProjectRepositories, @@ -59,10 +62,12 @@ impl Library { arch = Some(Architecture::from_str(arch_str)?); name = layer_name.to_owned(); } + let modified = mtime(&path)?; let layer_config = LayerConfig::load(&path)?; let layer_idx = *table .entry(name) - .or_insert_with(|| layers.push(Layer::default())); + .or_insert_with(|| layers.push(Layer::new(modified))); + layers[layer_idx].modified = layers[layer_idx].modified.max(modified); match arch { Some(arch) => { layers[layer_idx].arch_configs.insert(arch, layer_config); diff --git a/crates/rugpi-bakery/src/project/recipes.rs b/crates/rugpi-bakery/src/project/recipes.rs index b78c6e1..95a2683 100644 --- a/crates/rugpi-bakery/src/project/recipes.rs +++ b/crates/rugpi-bakery/src/project/recipes.rs @@ -13,6 +13,7 @@ use rugpi_common::Anyhow; use serde::{Deserialize, Serialize}; use super::repositories::RepositoryIdx; +use crate::caching::{mtime_recursive, ModificationTime}; /// Auxiliary data structure for loading recipes. #[derive(Debug)] @@ -40,6 +41,7 @@ impl RecipeLoader { /// Loads a recipe from the given path. pub fn load(&self, path: &Path) -> Anyhow { let path = path.to_path_buf(); + let modified = mtime_recursive(&path)?; let name = path .file_name() .ok_or_else(|| anyhow!("unable to determine recipe name from path `{path:?}`"))? @@ -61,6 +63,7 @@ impl RecipeLoader { steps.sort_by_key(|step| step.position); let recipe = Recipe { repository: self.repository, + modified, name, info, steps, @@ -76,6 +79,8 @@ impl RecipeLoader { /// A recipe. #[derive(Debug, Clone)] pub struct Recipe { + /// The lastest modification time of the recipe. + pub modified: ModificationTime, pub repository: RepositoryIdx, /// The name of the recipe. pub name: RecipeName, diff --git a/crates/rugpi-bakery/src/utils.rs b/crates/rugpi-bakery/src/utils.rs deleted file mode 100644 index 5098f1f..0000000 --- a/crates/rugpi-bakery/src/utils.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! Loop device interface. - -use std::path::{Path, PathBuf}; - -use rugpi_common::Anyhow; -use sha1::{Digest, Sha1}; -use url::Url; -use xscript::{run, Run}; - -pub fn download(url: &str) -> Anyhow { - let url = url.parse::()?; - let Some(file_name) = url.path_segments().and_then(|segments| segments.last()) else { - anyhow::bail!("unable to obtain file name from URL"); - }; - let file_extension = file_name.split_once('.').map(|(_, extension)| extension); - let mut url_hasher = Sha1::new(); - url_hasher.update(url.as_str().as_bytes()); - let url_hash = url_hasher.finalize(); - let mut cache_file_name = hex::encode(url_hash); - if let Some(extension) = file_extension { - cache_file_name.push('.'); - cache_file_name.push_str(extension); - } - let cache_file_path = Path::new(".rugpi/cache").join(cache_file_name); - if !cache_file_path.exists() { - std::fs::create_dir_all(".rugpi/cache")?; - run!(["wget", "-O", &cache_file_path, url.as_str()])?; - } - Ok(cache_file_path) -} - -pub fn sha1(string: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(string.as_bytes()); - hex::encode(hasher.finalize()) -}