diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 5dd16c9b1..8842a64b3 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -48,8 +48,8 @@ enum Commands { #[command(name = "copy")] Copy(MbtilesCopier), /// Apply diff file generated from 'copy' command - #[command(name = "apply-diff")] - ApplyDiff { + #[command(name = "apply-patch", alias = "apply-diff")] + ApplyPatch { /// MBTiles file to apply diff to src_file: PathBuf, /// Diff file @@ -100,7 +100,7 @@ async fn main_int() -> anyhow::Result<()> { Commands::Copy(opts) => { opts.run().await?; } - Commands::ApplyDiff { + Commands::ApplyPatch { src_file, diff_file, } => { @@ -150,7 +150,7 @@ mod tests { use clap::Parser; use martin_mbtiles::{CopyDuplicateMode, MbtilesCopier}; - use crate::Commands::{ApplyDiff, Copy, MetaGetValue, MetaSetValue, Validate}; + use crate::Commands::{ApplyPatch, Copy, MetaGetValue, MetaSetValue, Validate}; use crate::{Args, IntegrityCheckType}; #[test] @@ -410,7 +410,7 @@ mod tests { Args::parse_from(["mbtiles", "apply-diff", "src_file", "diff_file"]), Args { verbose: false, - command: ApplyDiff { + command: ApplyPatch { src_file: PathBuf::from("src_file"), diff_file: PathBuf::from("diff_file"), } diff --git a/martin-mbtiles/src/copier.rs b/martin-mbtiles/src/copier.rs index 3af7626a3..13ff828c5 100644 --- a/martin-mbtiles/src/copier.rs +++ b/martin-mbtiles/src/copier.rs @@ -10,8 +10,8 @@ use sqlite_hashes::rusqlite::params_from_iter; use sqlx::{query, Executor as _, Row, SqliteConnection}; use crate::errors::MbtResult; -use crate::mbtiles::MbtType; use crate::mbtiles::MbtType::{Flat, FlatWithHash, Normalized}; +use crate::mbtiles::{MbtType, MbtTypeCli}; use crate::queries::{ create_flat_tables, create_flat_with_hash_tables, create_normalized_tables, create_tiles_with_hash_view, detach_db, is_empty_database, @@ -36,7 +36,10 @@ pub struct MbtilesCopier { /// MBTiles file to write to pub dst_file: PathBuf, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source - #[cfg_attr(feature = "cli", arg(long, value_enum))] + #[cfg_attr(feature = "cli", arg(long = "dst_type", value_enum))] + pub dst_type_cli: Option, + /// Destination type with options + #[cfg_attr(feature = "cli", arg(skip))] pub dst_type: Option, /// Specify copying behaviour when tiles with duplicate (zoom_level, tile_column, tile_row) values are found #[cfg_attr(feature = "cli", arg(long, value_enum, default_value_t = CopyDuplicateMode::default()))] @@ -52,8 +55,12 @@ pub struct MbtilesCopier { pub zoom_levels: HashSet, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. - #[cfg_attr(feature = "cli", arg(long))] + #[cfg_attr(feature = "cli", arg(long, conflicts_with("apply_patch")))] pub diff_with_file: Option, + /// Compare source file with this file, and only copy non-identical tiles to destination. + /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. + #[cfg_attr(feature = "cli", arg(long, conflicts_with("diff_with_file")))] + pub apply_patch: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. #[cfg_attr(feature = "cli", arg(long))] pub skip_agg_tiles_hash: bool, @@ -105,11 +112,13 @@ impl MbtilesCopier { src_file: src_filepath, dst_file: dst_filepath, zoom_levels: HashSet::new(), + dst_type_cli: None, dst_type: None, on_duplicate: CopyDuplicateMode::Override, min_zoom: None, max_zoom: None, diff_with_file: None, + apply_patch: None, skip_agg_tiles_hash: false, } } @@ -117,10 +126,23 @@ impl MbtilesCopier { pub async fn run(self) -> MbtResult { MbtileCopierInt::new(self)?.run().await } + + pub(crate) fn dst_type(&self) -> Option { + self.dst_type.or_else(|| { + self.dst_type_cli.map(|t| match t { + MbtTypeCli::Flat => Flat, + MbtTypeCli::FlatWithHash => FlatWithHash, + MbtTypeCli::Normalized => Normalized { hash_view: true }, + }) + }) + } } impl MbtileCopierInt { pub fn new(options: MbtilesCopier) -> MbtResult { + if options.apply_patch.is_some() && options.diff_with_file.is_some() { + return Err(MbtError::CannotApplyPatchAndDiff); + } // We may want to resolve the files to absolute paths here, but will need to avoid various non-file cases if options.src_file == options.dst_file { return Err(MbtError::SameSourceAndDestination(options.src_file)); @@ -130,6 +152,12 @@ impl MbtileCopierInt { return Err(MbtError::SameDiffAndSourceOrDestination(options.src_file)); } } + if let Some(patch_file) = &options.apply_patch { + if options.src_file == *patch_file || options.dst_file == *patch_file { + return Err(MbtError::SameDiffAndSourceOrDestination(options.src_file)); + } + } + Ok(MbtileCopierInt { src_mbtiles: Mbtiles::new(&options.src_file)?, dst_mbtiles: Mbtiles::new(&options.dst_file)?, @@ -138,37 +166,43 @@ impl MbtileCopierInt { } pub async fn run(self) -> MbtResult { + let dif = match (&self.options.diff_with_file, &self.options.apply_patch) { + (Some(dif_file), None) | (None, Some(dif_file)) => { + let dif_mbt = Mbtiles::new(dif_file)?; + let dif_type = dif_mbt.open_and_detect_type().await?; + Some((dif_mbt, dif_type, dif_type)) + } + (Some(_), Some(_)) => unreachable!(), // validated in the Self::new + _ => None, + }; + // src and diff file connections are not needed later, as they will be attached to the dst file let src_mbt = &self.src_mbtiles; let dst_mbt = &self.dst_mbtiles; let src_type = src_mbt.open_and_detect_type().await?; - let dif = if let Some(dif_file) = &self.options.diff_with_file { - let dif_file = Mbtiles::new(dif_file)?; - let dif_type = dif_file.open_and_detect_type().await?; - Some((dif_file, dif_type)) - } else { - None - }; - let mut conn = dst_mbt.open_or_new().await?; let is_empty_db = is_empty_database(&mut conn).await?; src_mbt.attach_to(&mut conn, "sourceDb").await?; - let dst_type; - if let Some((dif_mbt, dif_type)) = &dif { + let dst_type: MbtType; + if let Some((dif_mbt, dif_type, _)) = &dif { if !is_empty_db { return Err(MbtError::NonEmptyTargetFile(self.options.dst_file)); } - dst_type = self.options.dst_type.unwrap_or(src_type); + dst_type = self.options.dst_type().unwrap_or(src_type); dif_mbt.attach_to(&mut conn, "diffDb").await?; let dif_path = dif_mbt.filepath(); - info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) into a new file {dst_mbt} ({dst_type})"); + if self.options.diff_with_file.is_some() { + info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) into a new file {dst_mbt} ({dst_type})"); + } else { + info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) into a new file {dst_mbt} ({dst_type})"); + } } else if is_empty_db { - dst_type = self.options.dst_type.unwrap_or(src_type); + dst_type = self.options.dst_type().unwrap_or(src_type); info!("Copying {src_mbt} ({src_type}) to a new file {dst_mbt} ({dst_type})"); } else { - dst_type = dst_mbt.detect_type(&mut conn).await?; + dst_type = self.validate_dst_type(dst_mbt.detect_type(&mut conn).await?)?; info!("Copying {src_mbt} ({src_type}) to an existing file {dst_mbt} ({dst_type})"); } @@ -176,8 +210,12 @@ impl MbtileCopierInt { self.init_new_schema(&mut conn, src_type, dst_type).await?; } - let select_from = if let Some((_, dif_type)) = &dif { - Self::get_select_from_with_diff(*dif_type, dst_type) + let select_from = if let Some((_, dif_type, _)) = &dif { + if self.options.diff_with_file.is_some() { + Self::get_select_from_with_diff(*dif_type, dst_type) + } else { + Self::get_select_from_apply_patch(src_type, *dif_type, dst_type) + } } else { Self::get_select_from(src_type, dst_type).to_string() }; @@ -215,12 +253,12 @@ impl MbtileCopierInt { debug!("Copying to {dst_type} with {sql} {query_args:?}"); rusqlite_conn.execute(&sql, params_from_iter(query_args))? } - Normalized => { + Normalized { .. } => { let sql = format!( " INSERT OR IGNORE INTO images (tile_id, tile_data) - SELECT hash as tile_id, tile_data + SELECT tile_hash as tile_id, tile_data FROM ({select_from})" ); debug!("Copying to {dst_type} with {sql} {query_args:?}"); @@ -230,7 +268,7 @@ impl MbtileCopierInt { " INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id) - SELECT zoom_level, tile_column, tile_row, hash as tile_id + SELECT zoom_level, tile_column, tile_row, tile_hash as tile_id FROM ({select_from} {sql_cond})" ); debug!("Copying to {dst_type} with {sql} {query_args:?}"); @@ -238,30 +276,54 @@ impl MbtileCopierInt { } }; - let sql = if self.options.diff_with_file.is_some() { - debug!("Copying metadata with 'INSERT {on_dupl}', taking into account diff file"); + let sql; + if dif.is_some() { // Insert all rows from diffDb.metadata if they do not exist or are different in sourceDb.metadata. // Also insert all names from sourceDb.metadata that do not exist in diffDb.metadata, with their value set to NULL. // Rename agg_tiles_hash to agg_tiles_hash_in_diff because agg_tiles_hash will be auto-added later - format!( + if self.options.diff_with_file.is_some() { + sql = format!( " INSERT {on_dupl} INTO metadata (name, value) - SELECT IIF(dif.name = '{AGG_TILES_HASH}','{AGG_TILES_HASH_IN_DIFF}', dif.name) as name, - dif.value as value - FROM diffDb.metadata AS dif LEFT JOIN sourceDb.metadata AS src - ON dif.name = src.name - WHERE (dif.value != src.value OR src.value ISNULL) - AND dif.name != '{AGG_TILES_HASH_IN_DIFF}' - UNION ALL - SELECT src.name as name, NULL as value - FROM sourceDb.metadata AS src LEFT JOIN diffDb.metadata AS dif - ON src.name = dif.name - WHERE dif.value ISNULL AND src.name NOT IN ('{AGG_TILES_HASH}', '{AGG_TILES_HASH_IN_DIFF}');" - ) + SELECT IIF(name = '{AGG_TILES_HASH}','{AGG_TILES_HASH_IN_DIFF}', name) as name + , value + FROM ( + SELECT COALESCE(difMD.name, srcMD.name) as name + , difMD.value as value + FROM sourceDb.metadata AS srcMD FULL JOIN diffDb.metadata AS difMD + ON srcMD.name = difMD.name + WHERE srcMD.value != difMD.value OR srcMD.value ISNULL OR difMD.value ISNULL + ) joinedMD + WHERE name != '{AGG_TILES_HASH_IN_DIFF}'" + ); + } else { + sql = format!( + " + INSERT {on_dupl} INTO metadata (name, value) + SELECT IIF(name = '{AGG_TILES_HASH_IN_DIFF}','{AGG_TILES_HASH}', name) as name + , value + FROM ( + SELECT COALESCE(srcMD.name, difMD.name) as name + , COALESCE(difMD.value, srcMD.value) as value + FROM sourceDb.metadata AS srcMD FULL JOIN diffDb.metadata AS difMD + ON srcMD.name = difMD.name + WHERE difMD.name ISNULL OR difMD.value NOTNULL + ) joinedMD + WHERE name != '{AGG_TILES_HASH}'" + ); + } + if self.options.diff_with_file.is_some() { + debug!("Copying metadata, taking into account diff file with {sql}"); + } else { + debug!("Copying metadata, and applying the diff file with {sql}"); + } } else { - debug!("Copying metadata with 'INSERT {on_dupl}'"); - format!("INSERT {on_dupl} INTO metadata SELECT name, value FROM sourceDb.metadata") - }; + sql = format!( + " + INSERT {on_dupl} INTO metadata SELECT name, value FROM sourceDb.metadata" + ); + debug!("Copying metadata with {sql}"); + } rusqlite_conn.execute(&sql, [])?; // SAFETY: must drop rusqlite_conn before handle_lock, or place the code since lock in a separate scope @@ -279,6 +341,25 @@ impl MbtileCopierInt { Ok(conn) } + /// Check if the detected destination file type matches the one given by the options + fn validate_dst_type(&self, dst_type: MbtType) -> MbtResult { + if let Some(cli) = self.options.dst_type() { + match (cli, dst_type) { + (Flat, Flat) + | (FlatWithHash, FlatWithHash) + | (Normalized { .. }, Normalized { .. }) => {} + (cli, dst) => { + return Err(MbtError::MismatchedTargetType( + self.options.dst_file.to_path_buf(), + dst, + cli, + )) + } + } + } + Ok(dst_type) + } + async fn init_new_schema( &self, conn: &mut SqliteConnection, @@ -317,11 +398,11 @@ impl MbtileCopierInt { match dst { Flat => create_flat_tables(&mut *conn).await?, FlatWithHash => create_flat_with_hash_tables(&mut *conn).await?, - Normalized => create_normalized_tables(&mut *conn).await?, + Normalized { .. } => create_normalized_tables(&mut *conn).await?, }; }; - if dst == Normalized { + if dst.is_normalized() { // Some normalized mbtiles files might not have this view, so even if src == dst, it might not exist create_tiles_with_hash_view(&mut *conn).await?; } @@ -338,7 +419,7 @@ impl MbtileCopierInt { let (main_table, tile_identifier) = match dst_type { Flat => ("tiles", "tile_data"), FlatWithHash => ("tiles_with_hash", "tile_data"), - Normalized => ("map", "tile_id"), + Normalized { .. } => ("map", "tile_id"), }; format!( @@ -356,24 +437,111 @@ impl MbtileCopierInt { } } + fn get_select_from_apply_patch( + src_type: MbtType, + dif_type: MbtType, + dst_type: MbtType, + ) -> String { + fn query_for_dst(frm_db: &'static str, frm_type: MbtType, to_type: MbtType) -> String { + match to_type { + Flat => format!("{frm_db}.tiles"), + FlatWithHash => match frm_type { + Flat => format!( + " + (SELECT zoom_level, tile_column, tile_row, tile_data, md5_hex(tile_data) AS tile_hash + FROM {frm_db}.tiles)" + ), + FlatWithHash => format!("{frm_db}.tiles_with_hash"), + Normalized { hash_view } => { + if hash_view { + format!("{frm_db}.tiles_with_hash") + } else { + format!( + " + (SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS tile_hash + FROM {frm_db}.map JOIN {frm_db}.images ON map.tile_id = images.tile_id)" + ) + } + } + }, + Normalized { .. } => match frm_type { + Flat => format!( + " + (SELECT zoom_level, tile_column, tile_row, tile_data, md5_hex(tile_data) AS tile_hash + FROM {frm_db}.tiles)" + ), + FlatWithHash => format!("{frm_db}.tiles_with_hash"), + Normalized { hash_view } => { + if hash_view { + format!("{frm_db}.tiles_with_hash") + } else { + format!( + " + (SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS tile_hash + FROM {frm_db}.map JOIN {frm_db}.images ON map.tile_id = images.tile_id)" + ) + } + } + }, + } + } + + let tile_hash_expr = if dst_type == Flat { + String::new() + } else { + fn get_tile_hash_expr(tbl: &str, typ: MbtType) -> String { + match typ { + Flat => format!("IIF({tbl}.tile_data ISNULL, NULL, md5_hex({tbl}.tile_data))"), + FlatWithHash => format!("{tbl}.tile_hash"), + Normalized { .. } => format!("{tbl}.tile_hash"), + } + } + + format!( + ", COALESCE({}, {}) as tile_hash", + get_tile_hash_expr("difTiles", dif_type), + get_tile_hash_expr("srcTiles", src_type) + ) + }; + + let src_tiles = query_for_dst("sourceDb", src_type, dst_type); + let diff_tiles = query_for_dst("diffDb", dif_type, dst_type); + + // Take dif tile_data if it is set, otherwise take the one from src + // Skip tiles if src and dif both have a matching index, but the dif tile_data is NULL + format!( + " + SELECT COALESCE(srcTiles.zoom_level, difTiles.zoom_level) as zoom_level + , COALESCE(srcTiles.tile_column, difTiles.tile_column) as tile_column + , COALESCE(srcTiles.tile_row, difTiles.tile_row) as tile_row + , COALESCE(difTiles.tile_data, srcTiles.tile_data) as tile_data + {tile_hash_expr} + FROM {src_tiles} AS srcTiles FULL JOIN {diff_tiles} AS difTiles + ON srcTiles.zoom_level = difTiles.zoom_level + AND srcTiles.tile_column = difTiles.tile_column + AND srcTiles.tile_row = difTiles.tile_row + WHERE (difTiles.zoom_level ISNULL OR difTiles.tile_data NOTNULL)" + ) + } + fn get_select_from_with_diff(dif_type: MbtType, dst_type: MbtType) -> String { - let hash_col_sql; + let tile_hash_expr; let diff_tiles; if dst_type == Flat { - hash_col_sql = ""; + tile_hash_expr = ""; diff_tiles = "diffDb.tiles"; } else { - hash_col_sql = match dif_type { - Flat => ", COALESCE(md5_hex(difTiles.tile_data), '') as hash", - FlatWithHash => ", COALESCE(difTiles.tile_hash, '') as hash", - Normalized => ", COALESCE(difTiles.hash, '') as hash", + tile_hash_expr = match dif_type { + Flat => ", COALESCE(md5_hex(difTiles.tile_data), '') as tile_hash", + FlatWithHash => ", COALESCE(difTiles.tile_hash, '') as tile_hash", + Normalized { .. } => ", COALESCE(difTiles.tile_hash, '') as tile_hash", }; diff_tiles = match dif_type { Flat => "diffDb.tiles", FlatWithHash => "diffDb.tiles_with_hash", - Normalized => { + Normalized { .. } => { " - (SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash + (SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS tile_hash FROM diffDb.map JOIN diffDb.images ON diffDb.map.tile_id = diffDb.images.tile_id)" } }; @@ -385,7 +553,7 @@ impl MbtileCopierInt { , COALESCE(srcTiles.tile_column, difTiles.tile_column) as tile_column , COALESCE(srcTiles.tile_row, difTiles.tile_row) as tile_row , difTiles.tile_data as tile_data - {hash_col_sql} + {tile_hash_expr} FROM sourceDb.tiles AS srcTiles FULL JOIN {diff_tiles} AS difTiles ON srcTiles.zoom_level = difTiles.zoom_level AND srcTiles.tile_column = difTiles.tile_column @@ -403,19 +571,19 @@ impl MbtileCopierInt { match src_type { Flat => { " - SELECT zoom_level, tile_column, tile_row, tile_data, md5_hex(tile_data) as hash + SELECT zoom_level, tile_column, tile_row, tile_data, md5_hex(tile_data) as tile_hash FROM sourceDb.tiles WHERE TRUE" } FlatWithHash => { " - SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash + SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash FROM sourceDb.tiles_with_hash WHERE TRUE" } - Normalized => { + Normalized { .. } => { " - SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash + SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS tile_hash FROM sourceDb.map JOIN sourceDb.images ON sourceDb.map.tile_id = sourceDb.images.tile_id WHERE TRUE" @@ -461,6 +629,11 @@ mod tests { use super::*; + const FLAT: Option = Some(MbtTypeCli::Flat); + const FLAT_WITH_HASH: Option = Some(MbtTypeCli::FlatWithHash); + const NORMALIZED: Option = Some(MbtTypeCli::Normalized); + const NORM_WITH_VIEW: MbtType = Normalized { hash_view: true }; + async fn get_one(conn: &mut SqliteConnection, sql: &str) -> T where for<'r> T: Decode<'r, Sqlite> + Type, @@ -471,11 +644,11 @@ mod tests { async fn verify_copy_all( src_filepath: PathBuf, dst_filepath: PathBuf, - dst_type: Option, + dst_type_cli: Option, expected_dst_type: MbtType, ) -> MbtResult<()> { let mut opt = MbtilesCopier::new(src_filepath.clone(), dst_filepath.clone()); - opt.dst_type = dst_type; + opt.dst_type_cli = dst_type_cli; let mut dst_conn = opt.run().await?; Mbtiles::new(src_filepath)? @@ -528,7 +701,7 @@ mod tests { let dst = PathBuf::from( "file:copy_flat_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); - verify_copy_all(src, dst, Some(Flat), Flat).await + verify_copy_all(src, dst, FLAT, Flat).await } #[actix_rt::test] @@ -536,7 +709,7 @@ mod tests { let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from("file:copy_flat_from_normalized_tables_mem_db?mode=memory&cache=shared"); - verify_copy_all(src, dst, Some(Flat), Flat).await + verify_copy_all(src, dst, FLAT, Flat).await } #[actix_rt::test] @@ -552,7 +725,7 @@ mod tests { let dst = PathBuf::from( "file:copy_flat_with_hash_from_flat_tables_mem_db?mode=memory&cache=shared", ); - verify_copy_all(src, dst, Some(FlatWithHash), FlatWithHash).await + verify_copy_all(src, dst, FLAT_WITH_HASH, FlatWithHash).await } #[actix_rt::test] @@ -561,14 +734,14 @@ mod tests { let dst = PathBuf::from( "file:copy_flat_with_hash_from_normalized_tables_mem_db?mode=memory&cache=shared", ); - verify_copy_all(src, dst, Some(FlatWithHash), FlatWithHash).await + verify_copy_all(src, dst, FLAT_WITH_HASH, FlatWithHash).await } #[actix_rt::test] async fn copy_normalized_tables() -> MbtResult<()> { let src = PathBuf::from("../tests/fixtures/mbtiles/geography-class-png.mbtiles"); let dst = PathBuf::from("file:copy_normalized_tables_mem_db?mode=memory&cache=shared"); - verify_copy_all(src, dst, None, Normalized).await + verify_copy_all(src, dst, None, NORM_WITH_VIEW).await } #[actix_rt::test] @@ -576,7 +749,7 @@ mod tests { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_normalized_from_flat_tables_mem_db?mode=memory&cache=shared"); - verify_copy_all(src, dst, Some(Normalized), Normalized).await + verify_copy_all(src, dst, NORMALIZED, NORM_WITH_VIEW).await } #[actix_rt::test] @@ -585,7 +758,7 @@ mod tests { let dst = PathBuf::from( "file:copy_normalized_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); - verify_copy_all(src, dst, Some(Normalized), Normalized).await + verify_copy_all(src, dst, NORMALIZED, NORM_WITH_VIEW).await } #[actix_rt::test] @@ -669,7 +842,7 @@ mod tests { .run() .await?; - verify_copy_all(src_file, dst, Some(Normalized), Flat).await + verify_copy_all(src_file, dst, NORMALIZED, Flat).await } #[actix_rt::test] diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs index f8b8b14ab..cb732b66e 100644 --- a/martin-mbtiles/src/errors.rs +++ b/martin-mbtiles/src/errors.rs @@ -3,6 +3,8 @@ use std::path::PathBuf; use martin_tile_utils::TileInfo; use sqlite_hashes::rusqlite; +use crate::MbtType; + #[derive(thiserror::Error, Debug)] pub enum MbtError { #[error("The source and destination MBTiles files are the same: {}", .0.display())] @@ -55,6 +57,12 @@ pub enum MbtError { #[error("Unexpected duplicate tiles found when copying")] DuplicateValues, + + #[error("Applying a patch while diffing is not supported")] + CannotApplyPatchAndDiff, + + #[error("The MBTiles file {0} has data of type {1}, but the desired type was set to {2}")] + MismatchedTargetType(PathBuf, MbtType, MbtType), } pub type MbtResult = Result; diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index d31dcb7af..434fb6a92 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -5,8 +5,8 @@ pub use errors::{MbtError, MbtResult}; mod mbtiles; pub use mbtiles::{ - calc_agg_tiles_hash, IntegrityCheckType, MbtType, Mbtiles, Metadata, AGG_TILES_HASH, - AGG_TILES_HASH_IN_DIFF, + calc_agg_tiles_hash, IntegrityCheckType, MbtType, MbtTypeCli, Mbtiles, Metadata, + AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF, }; mod pool; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index faf37a043..2faf2b894 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -22,7 +22,8 @@ use tilejson::{tilejson, Bounds, Center, TileJSON}; use crate::errors::{MbtError, MbtResult}; use crate::queries::{ - is_flat_tables_type, is_flat_with_hash_tables_type, is_normalized_tables_type, + has_tiles_with_hash, is_flat_tables_type, is_flat_with_hash_tables_type, + is_normalized_tables_type, }; use crate::MbtError::{ AggHashMismatch, AggHashValueNotFound, FailedIntegrityCheck, IncorrectTileHash, @@ -65,12 +66,30 @@ pub const AGG_TILES_HASH_IN_DIFF: &str = "agg_tiles_hash_after_apply"; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(ValueEnum))] -pub enum MbtType { +pub enum MbtTypeCli { Flat, FlatWithHash, Normalized, } +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, EnumDisplay)] +#[enum_display(case = "Kebab")] +pub enum MbtType { + Flat, + FlatWithHash, + Normalized { hash_view: bool }, +} + +impl MbtType { + pub fn is_normalized(&self) -> bool { + matches!(self, Self::Normalized { .. }) + } + + pub fn is_normalized_with_view(&self) -> bool { + matches!(self, Self::Normalized { hash_view: true }) + } +} + #[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(ValueEnum))] @@ -437,8 +456,10 @@ impl Mbtiles { for<'e> &'e mut T: SqliteExecutor<'e>, { debug!("Detecting MBTiles type for {self}"); - let mbt_type = if is_normalized_tables_type(&mut *conn).await? { - MbtType::Normalized + let typ = if is_normalized_tables_type(&mut *conn).await? { + MbtType::Normalized { + hash_view: has_tiles_with_hash(&mut *conn).await?, + } } else if is_flat_with_hash_tables_type(&mut *conn).await? { MbtType::FlatWithHash } else if is_flat_tables_type(&mut *conn).await? { @@ -447,10 +468,10 @@ impl Mbtiles { return Err(MbtError::InvalidDataFormat(self.filepath.clone())); }; - self.check_for_uniqueness_constraint(&mut *conn, mbt_type) + self.check_for_uniqueness_constraint(&mut *conn, typ) .await?; - Ok(mbt_type) + Ok(typ) } async fn check_for_uniqueness_constraint( @@ -464,7 +485,7 @@ impl Mbtiles { let table_name = match mbt_type { MbtType::Flat => "tiles", MbtType::FlatWithHash => "tiles_with_hash", - MbtType::Normalized => "map", + MbtType::Normalized { .. } => "map", }; let indexes = query("SELECT name FROM pragma_index_list(?) WHERE [unique] = 1") @@ -596,7 +617,7 @@ impl Mbtiles { WHERE expected != computed LIMIT 1;" } - MbtType::Normalized => { + MbtType::Normalized { .. } => { "SELECT expected, computed FROM ( SELECT upper(tile_id) AS expected, @@ -803,7 +824,7 @@ mod tests { let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await?; let res = mbt.detect_type(&mut conn).await?; - assert_eq!(res, MbtType::Normalized); + assert_eq!(res, MbtType::Normalized { hash_view: false }); let (mut conn, mbt) = open(":memory:").await?; let res = mbt.detect_type(&mut conn).await; diff --git a/martin-mbtiles/src/patcher.rs b/martin-mbtiles/src/patcher.rs index d56cb5cfd..94444a917 100644 --- a/martin-mbtiles/src/patcher.rs +++ b/martin-mbtiles/src/patcher.rs @@ -31,7 +31,7 @@ pub async fn apply_patch(src_file: PathBuf, patch_file: PathBuf) -> MbtResult<() SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM patchDb.tiles_with_hash" } - Normalized => { + Normalized { .. } => { " SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM patchDb.map LEFT JOIN patchDb.images @@ -58,7 +58,7 @@ pub async fn apply_patch(src_file: PathBuf, patch_file: PathBuf) -> MbtResult<() {select_from}" )], ), - Normalized => ( + Normalized { .. } => ( "map", vec![ format!( @@ -93,7 +93,7 @@ pub async fn apply_patch(src_file: PathBuf, patch_file: PathBuf) -> MbtResult<() .execute(&mut conn) .await?; - if src_type == Normalized { + if src_type.is_normalized() { debug!("Removing unused tiles from the images table (normalized schema)"); query("DELETE FROM images WHERE tile_id NOT IN (SELECT tile_id FROM map)") .execute(&mut conn) diff --git a/martin-mbtiles/tests/mbtiles.rs b/martin-mbtiles/tests/mbtiles.rs index 833452b9e..2e7ec641b 100644 --- a/martin-mbtiles/tests/mbtiles.rs +++ b/martin-mbtiles/tests/mbtiles.rs @@ -6,8 +6,10 @@ use ctor::ctor; use insta::{allow_duplicates, assert_display_snapshot}; use log::info; use martin_mbtiles::IntegrityCheckType::Off; -use martin_mbtiles::MbtType::{Flat, FlatWithHash, Normalized}; -use martin_mbtiles::{apply_patch, create_flat_tables, MbtResult, MbtType, Mbtiles, MbtilesCopier}; +use martin_mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized}; +use martin_mbtiles::{ + apply_patch, create_flat_tables, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier, +}; use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; use serde::Serialize; @@ -84,7 +86,7 @@ fn copier(src: &Mbtiles, dst: &Mbtiles) -> MbtilesCopier { MbtilesCopier::new(path(src), path(dst)) } -fn shorten(v: MbtType) -> &'static str { +fn shorten(v: MbtTypeCli) -> &'static str { match v { Flat => "flat", FlatWithHash => "hash", @@ -99,7 +101,7 @@ async fn open(file: &str) -> MbtResult<(Mbtiles, SqliteConnection)> { } macro_rules! open { - ($function:tt, $($arg:tt)*) => { + ($function:ident, $($arg:tt)*) => { open!(@"", $function, $($arg)*) }; (@$extra:literal, $function:tt, $($arg:tt)*) => {{ @@ -110,18 +112,18 @@ macro_rules! open { /// Create a new SQLite file of given type without agg_tiles_hash metadata value macro_rules! new_file_no_hash { - ($function:tt, $dst_type:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => {{ - new_file!(@true, $function, $dst_type, $sql_meta, $sql_data, $($arg)*) + ($function:ident, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => {{ + new_file!(@true, $function, $dst_type_cli, $sql_meta, $sql_data, $($arg)*) }}; } -/// Create a new SQLite file of type $dst_type with the given metadata and tiles +/// Create a new SQLite file of type $dst_type_cli with the given metadata and tiles macro_rules! new_file { - ($function:tt, $dst_type:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => { - new_file!(@false, $function, $dst_type, $sql_meta, $sql_data, $($arg)*) + ($function:ident, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => { + new_file!(@false, $function, $dst_type_cli, $sql_meta, $sql_data, $($arg)*) }; - (@$skip_agg:expr, $function:tt, $dst_type:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => {{ + (@$skip_agg:expr, $function:tt, $dst_type_cli:expr, $sql_meta:expr, $sql_data:expr, $($arg:tt)*) => {{ let (tmp_mbt, mut cn_tmp) = open!(@"temp", $function, $($arg)*); create_flat_tables(&mut cn_tmp).await.unwrap(); cn_tmp.execute($sql_data).await.unwrap(); @@ -129,7 +131,7 @@ macro_rules! new_file { let (dst_mbt, cn_dst) = open!($function, $($arg)*); let mut opt = copier(&tmp_mbt, &dst_mbt); - opt.dst_type = Some($dst_type); + opt.dst_type_cli = Some($dst_type_cli); opt.skip_agg_tiles_hash = $skip_agg; opt.run().await.unwrap(); @@ -146,61 +148,84 @@ macro_rules! assert_snapshot { }}; } -macro_rules! assert_dump { - ($connection:expr, $($arg:tt)*) => {{ - let dmp = dump($connection).await.unwrap(); - assert_snapshot!(&dmp, $($arg)*); - dmp - }}; +#[derive(Default)] +struct Databases( + HashMap<(&'static str, MbtTypeCli), (Vec, Mbtiles, SqliteConnection)>, +); + +impl Databases { + fn add( + &mut self, + name: &'static str, + typ: MbtTypeCli, + dump: Vec, + mbtiles: Mbtiles, + conn: SqliteConnection, + ) { + self.0.insert((name, typ), (dump, mbtiles, conn)); + } + fn dump(&self, name: &'static str, typ: MbtTypeCli) -> &Vec { + &self.0.get(&(name, typ)).unwrap().0 + } + fn mbtiles(&self, name: &'static str, typ: MbtTypeCli) -> &Mbtiles { + &self.0.get(&(name, typ)).unwrap().1 + } } -type Databases = HashMap<(&'static str, MbtType), Vec>; - +/// Generate a set of databases for testing, and validate them against snapshot files. +/// These dbs will be used by other tests to check against in various conditions. #[fixture] #[once] fn databases() -> Databases { futures::executor::block_on(async { - let mut result = HashMap::new(); + let mut result = Databases::default(); for &mbt_typ in &[Flat, FlatWithHash, Normalized] { let typ = shorten(mbt_typ); - let (raw_mbt, mut cn) = new_file_no_hash!( + let (raw_mbt, mut raw_cn) = new_file_no_hash!( databases, mbt_typ, METADATA_V1, TILES_V1, "{typ}__v1-no-hash" ); - let dmp = assert_dump!(&mut cn, "{typ}__v1-no-hash"); - result.insert(("v1_no_hash", mbt_typ), dmp); + let dmp = dump(&mut raw_cn).await.unwrap(); + assert_snapshot!(&dmp, "{typ}__v1-no-hash"); + result.add("v1_no_hash", mbt_typ, dmp, raw_mbt, raw_cn); let (v1_mbt, mut v1_cn) = open!(databases, "{typ}__v1"); - copier(&raw_mbt, &v1_mbt).run().await.unwrap(); - let dmp = assert_dump!(&mut v1_cn, "{typ}__v1"); + let raw_mbt = result.mbtiles("v1_no_hash", mbt_typ); + copier(raw_mbt, &v1_mbt).run().await.unwrap(); + let dmp = dump(&mut v1_cn).await.unwrap(); + assert_snapshot!(&dmp, "{typ}__v1"); let hash = v1_mbt.validate(Off, false).await.unwrap(); allow_duplicates! { assert_display_snapshot!(hash, @"096A8399D486CF443A5DF0CEC1AD8BB2"); } - result.insert(("v1", mbt_typ), dmp); + result.add("v1", mbt_typ, dmp, v1_mbt, v1_cn); let (v2_mbt, mut v2_cn) = new_file!(databases, mbt_typ, METADATA_V2, TILES_V2, "{typ}__v2"); - let dmp = assert_dump!(&mut v2_cn, "{typ}__v2"); + let dmp = dump(&mut v2_cn).await.unwrap(); + assert_snapshot!(&dmp, "{typ}__v2"); let hash = v2_mbt.validate(Off, false).await.unwrap(); allow_duplicates! { assert_display_snapshot!(hash, @"FE0D3090E8B4E89F2C755C08E8D76BEA"); } - result.insert(("v2", mbt_typ), dmp); + result.add("v2", mbt_typ, dmp, v2_mbt, v2_cn); let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif"); - let mut opt = copier(&v1_mbt, &dif_mbt); - opt.diff_with_file = Some(path(&v2_mbt)); + let v1_mbt = result.mbtiles("v1", mbt_typ); + let mut opt = copier(v1_mbt, &dif_mbt); + let v2_mbt = result.mbtiles("v2", mbt_typ); + opt.diff_with_file = Some(path(v2_mbt)); opt.run().await.unwrap(); - let dmp = assert_dump!(&mut dif_cn, "{typ}__dif"); + let dmp = dump(&mut dif_cn).await.unwrap(); + assert_snapshot!(&dmp, "{typ}__dif"); let hash = dif_mbt.validate(Off, false).await.unwrap(); allow_duplicates! { assert_display_snapshot!(hash, @"B86122579EDCDD4C51F3910894FCC1A1"); } - result.insert(("dif", mbt_typ), dmp); + result.add("dif", mbt_typ, dmp, dif_mbt, dif_cn); } result }) @@ -210,8 +235,8 @@ fn databases() -> Databases { #[trace] #[actix_rt::test] async fn convert( - #[values(Flat, FlatWithHash, Normalized)] frm_type: MbtType, - #[values(Flat, FlatWithHash, Normalized)] dst_type: MbtType, + #[values(Flat, FlatWithHash, Normalized)] frm_type: MbtTypeCli, + #[values(Flat, FlatWithHash, Normalized)] dst_type: MbtTypeCli, #[notrace] databases: &Databases, ) -> MbtResult<()> { let (frm, to) = (shorten(frm_type), shorten(dst_type)); @@ -219,23 +244,23 @@ async fn convert( let (frm_mbt, _frm_cn) = new_file!(convert, frm_type, METADATA_V1, TILES_V1, "{frm}-{to}"); let mut opt = copier(&frm_mbt, &mem); - opt.dst_type = Some(dst_type); + opt.dst_type_cli = Some(dst_type); let dmp = dump(&mut opt.run().await?).await?; - pretty_assert_eq!(databases.get(&("v1", dst_type)).unwrap(), &dmp); + pretty_assert_eq!(databases.dump("v1", dst_type), &dmp); let mut opt = copier(&frm_mbt, &mem); - opt.dst_type = Some(dst_type); + opt.dst_type_cli = Some(dst_type); opt.zoom_levels.insert(6); let z6only = dump(&mut opt.run().await?).await?; assert_snapshot!(z6only, "v1__z6__{frm}-{to}"); let mut opt = copier(&frm_mbt, &mem); - opt.dst_type = Some(dst_type); + opt.dst_type_cli = Some(dst_type); opt.min_zoom = Some(6); pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?); let mut opt = copier(&frm_mbt, &mem); - opt.dst_type = Some(dst_type); + opt.dst_type_cli = Some(dst_type); opt.min_zoom = Some(6); opt.max_zoom = Some(6); pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?); @@ -246,42 +271,45 @@ async fn convert( #[rstest] #[trace] #[actix_rt::test] -async fn diff_apply( - #[values(Flat, FlatWithHash, Normalized)] v1_type: MbtType, - #[values(Flat, FlatWithHash, Normalized)] v2_type: MbtType, - #[values(None, Some(Flat), Some(FlatWithHash), Some(Normalized))] dif_type: Option, +async fn diff_and_patch( + #[values(Flat, FlatWithHash, Normalized)] v1_type: MbtTypeCli, + #[values(Flat, FlatWithHash, Normalized)] v2_type: MbtTypeCli, + #[values(None, Some(Flat), Some(FlatWithHash), Some(Normalized))] dif_type: Option, #[notrace] databases: &Databases, ) -> MbtResult<()> { let (v1, v2) = (shorten(v1_type), shorten(v2_type)); let dif = dif_type.map(shorten).unwrap_or("dflt"); let prefix = format!("{v2}-{v1}={dif}"); - let (v1_mbt, _v1_cn) = new_file! {diff_apply, v1_type, METADATA_V1, TILES_V1, "{prefix}__v1"}; - let (v2_mbt, _v2_cn) = new_file! {diff_apply, v2_type, METADATA_V2, TILES_V2, "{prefix}__v2"}; - let (dif_mbt, mut dif_cn) = open!(diff_apply, "{prefix}__dif"); + let v1_mbt = databases.mbtiles("v1", v1_type); + let v2_mbt = databases.mbtiles("v2", v2_type); + let (dif_mbt, mut dif_cn) = open!(diff_and_patchdiff_and_patch, "{prefix}__dif"); info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)"); - let mut opt = copier(&v1_mbt, &dif_mbt); - opt.diff_with_file = Some(path(&v2_mbt)); + let mut opt = copier(v1_mbt, &dif_mbt); + opt.diff_with_file = Some(path(v2_mbt)); if let Some(dif_type) = dif_type { - opt.dst_type = Some(dif_type); + opt.dst_type_cli = Some(dif_type); } opt.run().await?; pretty_assert_eq!( &dump(&mut dif_cn).await?, - databases - .get(&("dif", dif_type.unwrap_or(v1_type))) - .unwrap() + databases.dump("dif", dif_type.unwrap_or(v1_type)) ); for target_type in &[Flat, FlatWithHash, Normalized] { let trg = shorten(*target_type); let prefix = format!("{prefix}__to__{trg}"); - let expected_v2 = databases.get(&("v2", *target_type)).unwrap(); + let expected_v2 = databases.dump("v2", *target_type); info!("TEST: Applying the difference (v2-v1=diff) to v1, should get v2"); - let (tar1_mbt, mut tar1_cn) = - new_file! {diff_apply, *target_type, METADATA_V1, TILES_V1, "{prefix}__v1"}; + let (tar1_mbt, mut tar1_cn) = new_file!( + diff_and_patch, + *target_type, + METADATA_V1, + TILES_V1, + "{prefix}__v1" + ); apply_patch(path(&tar1_mbt), path(&dif_mbt)).await?; let hash_v1 = tar1_mbt.validate(Off, false).await?; allow_duplicates! { @@ -292,7 +320,7 @@ async fn diff_apply( info!("TEST: Applying the difference (v2-v1=diff) to v2, should not modify it"); let (tar2_mbt, mut tar2_cn) = - new_file! {diff_apply, *target_type, METADATA_V2, TILES_V2, "{prefix}__v2"}; + new_file! {diff_and_patch, *target_type, METADATA_V2, TILES_V2, "{prefix}__v2"}; apply_patch(path(&tar2_mbt), path(&dif_mbt)).await?; let hash_v2 = tar2_mbt.validate(Off, false).await?; allow_duplicates! { @@ -305,17 +333,56 @@ async fn diff_apply( Ok(()) } -// /// A simple tester to run specific values -// #[actix_rt::test] -// async fn test_one() { -// let dif_type = FlatWithHash; -// let src_type = Flat; -// let dst_type = Some(Normalized); -// let db = databases(); -// -// diff_apply(src_type, dif_type, dst_type, &db).await.unwrap(); -// panic!() -// } +#[rstest] +#[trace] +#[actix_rt::test] +async fn patch_on_copy( + #[values(Flat, FlatWithHash, Normalized)] v1_type: MbtTypeCli, + #[values(Flat, FlatWithHash, Normalized)] dif_type: MbtTypeCli, + #[values(None, Some(Flat), Some(FlatWithHash), Some(Normalized))] v2_type: Option, + #[notrace] databases: &Databases, +) -> MbtResult<()> { + let (v1, dif) = (shorten(v1_type), shorten(dif_type)); + let v2 = v2_type.map(shorten).unwrap_or("dflt"); + let prefix = format!("{v1}+{dif}={v2}"); + + let v1_mbt = databases.mbtiles("v1", v1_type); + let dif_mbt = databases.mbtiles("dif", dif_type); + let (v2_mbt, mut v2_cn) = open!(patch_on_copy, "{prefix}__v2"); + + info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)"); + let mut opt = copier(v1_mbt, &v2_mbt); + opt.apply_patch = Some(path(dif_mbt)); + if let Some(v2_type) = v2_type { + opt.dst_type_cli = Some(v2_type); + } + opt.run().await?; + pretty_assert_eq!( + &dump(&mut v2_cn).await?, + databases.dump("v2", v2_type.unwrap_or(v1_type)) + ); + + Ok(()) +} + +/// A simple tester to run specific values +#[actix_rt::test] +#[ignore] +async fn test_one() { + let src_type = FlatWithHash; + let dif_type = FlatWithHash; + // let dst_type = Some(FlatWithHash); + let dst_type = None; + let db = databases(); + + diff_and_patch(src_type, dif_type, dst_type, &db) + .await + .unwrap(); + patch_on_copy(src_type, dif_type, dst_type, &db) + .await + .unwrap(); + panic!("ALWAYS FAIL - this test is for debugging only, and should be disabled"); +} #[derive(Debug, sqlx::FromRow, Serialize, PartialEq)] struct SqliteEntry { diff --git a/tests/expected/mbtiles/help.txt b/tests/expected/mbtiles/help.txt index a7f0896af..55eb51149 100644 --- a/tests/expected/mbtiles/help.txt +++ b/tests/expected/mbtiles/help.txt @@ -3,13 +3,13 @@ A utility to work with .mbtiles file content Usage: mbtiles Commands: - meta-all Prints all values in the metadata table in a free-style, unstable YAML format - meta-get Gets a single value from the MBTiles metadata table - meta-set Sets a single value in the MBTiles' file metadata table or deletes it if no value - copy Copy tiles from one mbtiles file to another - apply-diff Apply diff file generated from 'copy' command - validate Validate tile data if hash of tile data exists in file - help Print this message or the help of the given subcommand(s) + meta-all Prints all values in the metadata table in a free-style, unstable YAML format + meta-get Gets a single value from the MBTiles metadata table + meta-set Sets a single value in the MBTiles' file metadata table or deletes it if no value + copy Copy tiles from one mbtiles file to another + apply-patch Apply diff file generated from 'copy' command + validate Validate tile data if hash of tile data exists in file + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help