mbtiles: Add --apply-patch to copy, rename apply-diff

* `mbtiles apply-diff` is now `apply-patch` (old name is still supported) * `mbtiles copy` can now take `--apply-patch <file>` to apply the patch while copying from source to destination. This way, the source file will remain unmodified.
maplibre · Oct 16, 2023 · 9a05f56 · 9a05f56
1 parent e2a26fe
commit 9a05f56
Show file tree

Hide file tree

Showing 8 changed files with 428 additions and 159 deletions.
diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs
@@ -48,8 +48,8 @@ enum Commands {
     #[command(name = "copy")]
     Copy(MbtilesCopier),
     /// Apply diff file generated from 'copy' command
-    #[command(name = "apply-diff")]
-    ApplyDiff {
+    #[command(name = "apply-patch", alias = "apply-diff")]
+    ApplyPatch {
         /// MBTiles file to apply diff to
         src_file: PathBuf,
         /// Diff file
@@ -100,7 +100,7 @@ async fn main_int() -> anyhow::Result<()> {
         Commands::Copy(opts) => {
             opts.run().await?;
         }
-        Commands::ApplyDiff {
+        Commands::ApplyPatch {
             src_file,
             diff_file,
         } => {
@@ -150,7 +150,7 @@ mod tests {
     use clap::Parser;
     use martin_mbtiles::{CopyDuplicateMode, MbtilesCopier};
 
-    use crate::Commands::{ApplyDiff, Copy, MetaGetValue, MetaSetValue, Validate};
+    use crate::Commands::{ApplyPatch, Copy, MetaGetValue, MetaSetValue, Validate};
     use crate::{Args, IntegrityCheckType};
 
     #[test]
@@ -410,7 +410,7 @@ mod tests {
             Args::parse_from(["mbtiles", "apply-diff", "src_file", "diff_file"]),
             Args {
                 verbose: false,
-                command: ApplyDiff {
+                command: ApplyPatch {
                     src_file: PathBuf::from("src_file"),
                     diff_file: PathBuf::from("diff_file"),
                 }

diff --git a/martin-mbtiles/src/copier.rs b/martin-mbtiles/src/copier.rs
diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs
@@ -3,6 +3,8 @@ use std::path::PathBuf;
 use martin_tile_utils::TileInfo;
 use sqlite_hashes::rusqlite;
 
+use crate::MbtType;
+
 #[derive(thiserror::Error, Debug)]
 pub enum MbtError {
     #[error("The source and destination MBTiles files are the same: {}", .0.display())]
@@ -55,6 +57,12 @@ pub enum MbtError {
 
     #[error("Unexpected duplicate tiles found when copying")]
     DuplicateValues,
+
+    #[error("Applying a patch while diffing is not supported")]
+    CannotApplyPatchAndDiff,
+
+    #[error("The MBTiles file {0} has data of type {1}, but the desired type was set to {2}")]
+    MismatchedTargetType(PathBuf, MbtType, MbtType),
 }
 
 pub type MbtResult<T> = Result<T, MbtError>;
diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs
@@ -5,8 +5,8 @@ pub use errors::{MbtError, MbtResult};
 
 mod mbtiles;
 pub use mbtiles::{
-    calc_agg_tiles_hash, IntegrityCheckType, MbtType, Mbtiles, Metadata, AGG_TILES_HASH,
-    AGG_TILES_HASH_IN_DIFF,
+    calc_agg_tiles_hash, IntegrityCheckType, MbtType, MbtTypeCli, Mbtiles, Metadata,
+    AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF,
 };
 
 mod pool;

diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs
@@ -22,7 +22,8 @@ use tilejson::{tilejson, Bounds, Center, TileJSON};
 
 use crate::errors::{MbtError, MbtResult};
 use crate::queries::{
-    is_flat_tables_type, is_flat_with_hash_tables_type, is_normalized_tables_type,
+    has_tiles_with_hash, is_flat_tables_type, is_flat_with_hash_tables_type,
+    is_normalized_tables_type,
 };
 use crate::MbtError::{
     AggHashMismatch, AggHashValueNotFound, FailedIntegrityCheck, IncorrectTileHash,
@@ -65,12 +66,30 @@ pub const AGG_TILES_HASH_IN_DIFF: &str = "agg_tiles_hash_after_apply";
 #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, EnumDisplay)]
 #[enum_display(case = "Kebab")]
 #[cfg_attr(feature = "cli", derive(ValueEnum))]
-pub enum MbtType {
+pub enum MbtTypeCli {
     Flat,
     FlatWithHash,
     Normalized,
 }
 
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, EnumDisplay)]
+#[enum_display(case = "Kebab")]
+pub enum MbtType {
+    Flat,
+    FlatWithHash,
+    Normalized { hash_view: bool },
+}
+
+impl MbtType {
+    pub fn is_normalized(&self) -> bool {
+        matches!(self, Self::Normalized { .. })
+    }
+
+    pub fn is_normalized_with_view(&self) -> bool {
+        matches!(self, Self::Normalized { hash_view: true })
+    }
+}
+
 #[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)]
 #[enum_display(case = "Kebab")]
 #[cfg_attr(feature = "cli", derive(ValueEnum))]
@@ -437,8 +456,10 @@ impl Mbtiles {
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
         debug!("Detecting MBTiles type for {self}");
-        let mbt_type = if is_normalized_tables_type(&mut *conn).await? {
-            MbtType::Normalized
+        let typ = if is_normalized_tables_type(&mut *conn).await? {
+            MbtType::Normalized {
+                hash_view: has_tiles_with_hash(&mut *conn).await?,
+            }
         } else if is_flat_with_hash_tables_type(&mut *conn).await? {
             MbtType::FlatWithHash
         } else if is_flat_tables_type(&mut *conn).await? {
@@ -447,10 +468,10 @@ impl Mbtiles {
             return Err(MbtError::InvalidDataFormat(self.filepath.clone()));
         };
 
-        self.check_for_uniqueness_constraint(&mut *conn, mbt_type)
+        self.check_for_uniqueness_constraint(&mut *conn, typ)
             .await?;
 
-        Ok(mbt_type)
+        Ok(typ)
     }
 
     async fn check_for_uniqueness_constraint<T>(
@@ -464,7 +485,7 @@ impl Mbtiles {
         let table_name = match mbt_type {
             MbtType::Flat => "tiles",
             MbtType::FlatWithHash => "tiles_with_hash",
-            MbtType::Normalized => "map",
+            MbtType::Normalized { .. } => "map",
         };
 
         let indexes = query("SELECT name FROM pragma_index_list(?) WHERE [unique] = 1")
@@ -596,7 +617,7 @@ impl Mbtiles {
                 WHERE expected != computed
                 LIMIT 1;"
             }
-            MbtType::Normalized => {
+            MbtType::Normalized { .. } => {
                 "SELECT expected, computed FROM (
                     SELECT
                         upper(tile_id) AS expected,
@@ -803,7 +824,7 @@ mod tests {
 
         let (mut conn, mbt) = open("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles").await?;
         let res = mbt.detect_type(&mut conn).await?;
-        assert_eq!(res, MbtType::Normalized);
+        assert_eq!(res, MbtType::Normalized { hash_view: false });
 
         let (mut conn, mbt) = open(":memory:").await?;
         let res = mbt.detect_type(&mut conn).await;

diff --git a/martin-mbtiles/src/patcher.rs b/martin-mbtiles/src/patcher.rs
@@ -31,7 +31,7 @@ pub async fn apply_patch(src_file: PathBuf, patch_file: PathBuf) -> MbtResult<()
         SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash
         FROM patchDb.tiles_with_hash"
             }
-            Normalized => {
+            Normalized { .. } => {
                 "
         SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash
         FROM patchDb.map LEFT JOIN patchDb.images
@@ -58,7 +58,7 @@ pub async fn apply_patch(src_file: PathBuf, patch_file: PathBuf) -> MbtResult<()
     {select_from}"
             )],
         ),
-        Normalized => (
+        Normalized { .. } => (
             "map",
             vec![
                 format!(
@@ -93,7 +93,7 @@ pub async fn apply_patch(src_file: PathBuf, patch_file: PathBuf) -> MbtResult<()
     .execute(&mut conn)
     .await?;
 
-    if src_type == Normalized {
+    if src_type.is_normalized() {
         debug!("Removing unused tiles from the images table (normalized schema)");
         query("DELETE FROM images WHERE tile_id NOT IN (SELECT tile_id FROM map)")
             .execute(&mut conn)