Skip to content

Commit

Permalink
feat: use mtime to detect changes
Browse files Browse the repository at this point in the history
  • Loading branch information
koehlma committed Jan 9, 2024
1 parent c6f65e4 commit b3a69cb
Show file tree
Hide file tree
Showing 8 changed files with 133 additions and 59 deletions.
29 changes: 21 additions & 8 deletions crates/rugpi-bakery/src/bake/customize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@ use rugpi_common::{mount::Mounted, Anyhow};
use tempfile::tempdir;
use xscript::{cmd, run, vars, ParentEnv, Run};

use crate::project::{
config::Architecture,
layers::LayerConfig,
library::Library,
recipes::{Recipe, StepKind},
Project,
use crate::{
caching::mtime,
project::{
config::Architecture,
layers::{Layer, LayerConfig},
library::Library,
recipes::{Recipe, StepKind},
Project,
},
};

/// The arguments of the `customize` command.
Expand All @@ -34,13 +37,23 @@ pub struct CustomizeTask {
pub fn customize(
project: &Project,
arch: Architecture,
layer: &LayerConfig,
layer: &Layer,
src: &Path,
target: &Path,
) -> Anyhow<()> {
let library = project.load_library()?;
// Collect the recipes to apply.
let jobs = recipe_schedule(layer, &library)?;
let config = layer.config(arch).unwrap();
let jobs = recipe_schedule(config, &library)?;
let last_modified = jobs
.iter()
.map(|job| job.recipe.modified)
.max()
.unwrap()
.max(layer.modified);
if target.exists() && last_modified < mtime(target)? {
return Ok(());
}
// Prepare system chroot.
let root_dir = tempdir()?;
let root_dir_path = root_dir.path();
Expand Down
19 changes: 8 additions & 11 deletions crates/rugpi-bakery/src/bake/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ use tempfile::tempdir;
use xscript::{run, Run};

use crate::{
caching::{download, sha1},
project::{config::Architecture, Project},
utils::{download, sha1},
};

pub mod customize;
Expand All @@ -28,13 +28,12 @@ pub fn bake_image(project: &Project, image: &str, output: &Path) -> Anyhow<()> {
image::make_image(image_config, &baked_layer, output)
}

pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow<PathBuf> {
pub fn bake_layer(project: &Project, arch: Architecture, layer_name: &str) -> Anyhow<PathBuf> {
let library = project.load_library()?;
let layer_config = &library.layers[library
.lookup_layer(library.repositories.root_repository, layer)
.unwrap()]
.config(arch)
.unwrap();
let layer = &library.layers[library
.lookup_layer(library.repositories.root_repository, layer_name)
.unwrap()];
let layer_config = layer.config(arch).unwrap();
if let Some(url) = &layer_config.url {
let layer_id = sha1(url);
let system_tar = project
Expand All @@ -46,17 +45,15 @@ pub fn bake_layer(project: &Project, arch: Architecture, layer: &str) -> Anyhow<
Ok(system_tar)
} else if let Some(parent) = &layer_config.parent {
let src = bake_layer(project, arch, parent)?;
let mut layer_string = layer.to_owned();
let mut layer_string = layer_name.to_owned();
layer_string.push('.');
layer_string.push_str(arch.as_str());
let layer_id = sha1(&layer_string);
let target = project
.dir
.join(format!(".rugpi/layers/{layer_id}/system.tar"));
fs::create_dir_all(target.parent().unwrap()).ok();
if !target.exists() {
customize::customize(project, arch, layer_config, &src, &target)?;
}
customize::customize(project, arch, layer, &src, &target)?;
Ok(target)
} else {
bail!("invalid layer configuration")
Expand Down
80 changes: 80 additions & 0 deletions crates/rugpi-bakery/src/caching.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
//! Utilities for caching.
use std::{
fs, io,
path::{Path, PathBuf},
time::SystemTime,
};

use rugpi_common::Anyhow;
use serde::{Deserialize, Serialize};
use sha1::{Digest, Sha1};
use url::Url;
use xscript::{run, Run};

pub fn download(url: &str) -> Anyhow<PathBuf> {
let url = url.parse::<Url>()?;
let Some(file_name) = url.path_segments().and_then(|segments| segments.last()) else {
anyhow::bail!("unable to obtain file name from URL");
};
let file_extension = file_name.split_once('.').map(|(_, extension)| extension);
let mut url_hasher = Sha1::new();
url_hasher.update(url.as_str().as_bytes());
let url_hash = url_hasher.finalize();
let mut cache_file_name = hex::encode(url_hash);
if let Some(extension) = file_extension {
cache_file_name.push('.');
cache_file_name.push_str(extension);
}
let cache_file_path = Path::new(".rugpi/cache").join(cache_file_name);
if !cache_file_path.exists() {
std::fs::create_dir_all(".rugpi/cache")?;
run!(["wget", "-O", &cache_file_path, url.as_str()])?;
}
Ok(cache_file_path)
}

pub fn sha1(string: &str) -> String {
let mut hasher = Sha1::new();
hasher.update(string.as_bytes());
hex::encode(hasher.finalize())
}

/// Modification time in seconds since the UNIX epoch.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ModificationTime(u64);

impl ModificationTime {
/// Extract the modification time from the provided filesystem metadata.
fn from_metadata(metadata: &fs::Metadata) -> Result<Self, io::Error> {
metadata.modified().map(|modified| {
ModificationTime(
modified
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs(),
)
})
}
}

/// The modification time of the given path.
pub fn mtime(path: &Path) -> Result<ModificationTime, io::Error> {
fs::metadata(path).and_then(|metadata| ModificationTime::from_metadata(&metadata))
}

/// Recursively scans a path and return the latest modification time.
pub fn mtime_recursive(path: &Path) -> Result<ModificationTime, io::Error> {
let mut time = mtime(path)?;
if path.is_dir() {
for entry in fs::read_dir(path)? {
let entry = entry?;
time = time.max(if entry.file_type()?.is_dir() {
mtime_recursive(&entry.path())?
} else {
ModificationTime::from_metadata(&entry.metadata()?)?
});
}
}
Ok(time)
}
2 changes: 1 addition & 1 deletion crates/rugpi-bakery/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ use project::{config::Architecture, repositories::Source, ProjectLoader};
use rugpi_common::Anyhow;

pub mod bake;
pub mod caching;
pub mod idx_vec;
pub mod project;
pub mod utils;

#[derive(Debug, Parser)]
pub struct Args {
Expand Down
12 changes: 11 additions & 1 deletion crates/rugpi-bakery/src/project/layers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use super::{
config::Architecture,
recipes::{ParameterValue, RecipeName},
};
use crate::caching::ModificationTime;

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
Expand Down Expand Up @@ -42,13 +43,22 @@ impl LayerConfig {
}
}

#[derive(Debug, Default)]
#[derive(Debug)]
pub struct Layer {
pub modified: ModificationTime,
pub default_config: Option<LayerConfig>,
pub arch_configs: HashMap<Architecture, LayerConfig>,
}

impl Layer {
pub fn new(modified: ModificationTime) -> Self {
Self {
modified,
default_config: None,
arch_configs: HashMap::new(),
}
}

/// The layer configuration for the given architecture.
pub fn config(&self, arch: Architecture) -> Option<&LayerConfig> {
self.arch_configs
Expand Down
9 changes: 7 additions & 2 deletions crates/rugpi-bakery/src/project/library.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ use super::{
recipes::{Recipe, RecipeLoader},
repositories::{ProjectRepositories, RepositoryIdx},
};
use crate::idx_vec::{new_idx_type, IdxVec};
use crate::{
caching::mtime,
idx_vec::{new_idx_type, IdxVec},
};

pub struct Library {
pub repositories: ProjectRepositories,
Expand Down Expand Up @@ -59,10 +62,12 @@ impl Library {
arch = Some(Architecture::from_str(arch_str)?);
name = layer_name.to_owned();
}
let modified = mtime(&path)?;
let layer_config = LayerConfig::load(&path)?;
let layer_idx = *table
.entry(name)
.or_insert_with(|| layers.push(Layer::default()));
.or_insert_with(|| layers.push(Layer::new(modified)));
layers[layer_idx].modified = layers[layer_idx].modified.max(modified);
match arch {
Some(arch) => {
layers[layer_idx].arch_configs.insert(arch, layer_config);
Expand Down
5 changes: 5 additions & 0 deletions crates/rugpi-bakery/src/project/recipes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use rugpi_common::Anyhow;
use serde::{Deserialize, Serialize};

use super::repositories::RepositoryIdx;
use crate::caching::{mtime_recursive, ModificationTime};

/// Auxiliary data structure for loading recipes.
#[derive(Debug)]
Expand Down Expand Up @@ -40,6 +41,7 @@ impl RecipeLoader {
/// Loads a recipe from the given path.
pub fn load(&self, path: &Path) -> Anyhow<Recipe> {
let path = path.to_path_buf();
let modified = mtime_recursive(&path)?;
let name = path
.file_name()
.ok_or_else(|| anyhow!("unable to determine recipe name from path `{path:?}`"))?
Expand All @@ -61,6 +63,7 @@ impl RecipeLoader {
steps.sort_by_key(|step| step.position);
let recipe = Recipe {
repository: self.repository,
modified,
name,
info,
steps,
Expand All @@ -76,6 +79,8 @@ impl RecipeLoader {
/// A recipe.
#[derive(Debug, Clone)]
pub struct Recipe {
/// The lastest modification time of the recipe.
pub modified: ModificationTime,
pub repository: RepositoryIdx,
/// The name of the recipe.
pub name: RecipeName,
Expand Down
36 changes: 0 additions & 36 deletions crates/rugpi-bakery/src/utils.rs

This file was deleted.

0 comments on commit b3a69cb

Please sign in to comment.