Multiple mbtiles and martin-cp fixes (#1083)

* BREAKING: `martin-cp` will now set `format=pbf` instead of `mvt`. This
is what QGIS and possibly others expect, and this is what tools like
tilelive generates.
* `martin-cp` sets `minzoom` and `maxzoom` metadata values based on the
zoom parameters
* Add `mbtiles meta-update` command to refresh zoom levels based on the
present tiles.

Partially addresses items in #1081

martin-tile-utils/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "martin-tile-utils"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server."
 keywords = ["maps", "tiles", "mvt", "tileserver"]

martin-tile-utils/src/lib.rs

@@ -35,6 +35,20 @@ impl Format {
         })
     }
 
+    /// Get the `format` value as it should be stored in the `MBTiles` metadata table
+    #[must_use]
+    pub fn metadata_format_value(&self) -> &'static str {
+        match *self {
+            Self::Gif => "gif",
+            Self::Jpeg => "jpeg",
+            Self::Json => "json",
+            // QGIS uses `pbf` instead of `mvt` for some reason
+            Self::Mvt => "pbf",
+            Self::Png => "png",
+            Self::Webp => "webp",
+        }
+    }
+
     #[must_use]
     pub fn content_type(&self) -> &str {
         match *self {

martin/src/bin/martin-cp.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fmt::{Debug, Display, Formatter};
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicU64, Ordering};
@@ -153,20 +154,12 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> {
 
 fn compute_tile_ranges(args: &CopyArgs) -> Vec<TileRect> {
     let mut ranges = Vec::new();
-    let mut zooms_vec = Vec::new();
-    let zooms = if let Some(max_zoom) = args.max_zoom {
-        let min_zoom = args.min_zoom.unwrap_or(0);
-        zooms_vec.extend(min_zoom..=max_zoom);
-        &zooms_vec
-    } else {
-        &args.zoom_levels
-    };
     let boxes = if args.bbox.is_empty() {
         vec![Bounds::MAX_TILED]
     } else {
         args.bbox.clone()
     };
-    for zoom in zooms {
+    for zoom in get_zooms(args).iter() {
         for bbox in &boxes {
             let (min_x, min_y, max_x, max_y) =
                 bbox_to_xyz(bbox.left, bbox.bottom, bbox.right, bbox.top, *zoom);
@@ -179,6 +172,17 @@ fn compute_tile_ranges(args: &CopyArgs) -> Vec<TileRect> {
     ranges
 }
 
+fn get_zooms(args: &CopyArgs) -> Cow<Vec<u8>> {
+    if let Some(max_zoom) = args.max_zoom {
+        let mut zooms_vec = Vec::new();
+        let min_zoom = args.min_zoom.unwrap_or(0);
+        zooms_vec.extend(min_zoom..=max_zoom);
+        Cow::Owned(zooms_vec)
+    } else {
+        Cow::Borrowed(&args.zoom_levels)
+    }
+}
+
 struct TileXyz {
     xyz: TileCoord,
     data: TileData,
@@ -281,7 +285,7 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()>
     } else {
         CopyDuplicateMode::Override
     };
-    let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, args.mbt_type).await?;
+    let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?;
     let query = args.url_query.as_deref();
     let req = TestRequest::default()
         .insert_header((ACCEPT_ENCODING, args.encoding.as_str()))
@@ -371,10 +375,10 @@ async fn init_schema(
     conn: &mut SqliteConnection,
     sources: &[&dyn Source],
     tile_info: TileInfo,
-    mbt_type: Option<MbtTypeCli>,
+    args: &CopyArgs,
 ) -> Result<MbtType, MartinError> {
     Ok(if is_empty_database(&mut *conn).await? {
-        let mbt_type = match mbt_type.unwrap_or(MbtTypeCli::Normalized) {
+        let mbt_type = match args.mbt_type.unwrap_or(MbtTypeCli::Normalized) {
             MbtTypeCli::Flat => MbtType::Flat,
             MbtTypeCli::FlatWithHash => MbtType::FlatWithHash,
             MbtTypeCli::Normalized => MbtType::Normalized { hash_view: true },
@@ -383,12 +387,19 @@ async fn init_schema(
         let mut tj = merge_tilejson(sources, String::new());
         tj.other.insert(
             "format".to_string(),
-            serde_json::Value::String(tile_info.format.to_string()),
+            serde_json::Value::String(tile_info.format.metadata_format_value().to_string()),
         );
         tj.other.insert(
             "generator".to_string(),
             serde_json::Value::String(format!("martin-cp v{VERSION}")),
         );
+        let zooms = get_zooms(args);
+        if let Some(min_zoom) = zooms.iter().min() {
+            tj.minzoom = Some(*min_zoom);
+        }
+        if let Some(max_zoom) = zooms.iter().max() {
+            tj.maxzoom = Some(*max_zoom);
+        }
         mbt.insert_metadata(&mut *conn, &tj).await?;
         mbt_type
     } else {

mbtiles/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "mbtiles"
-version = "0.8.4"
+version = "0.8.5"
 authors = ["Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics."
 keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"]

mbtiles/src/bin/mbtiles.rs

@@ -63,6 +63,12 @@ enum Commands {
         /// Diff file
         diff_file: PathBuf,
     },
+    /// Update metadata to match the content of the file
+    #[command(name = "meta-update", alias = "update-meta")]
+    UpdateMetadata {
+        /// MBTiles file to validate
+        file: PathBuf,
+    },
     /// Validate tile data if hash of tile data exists in file
     #[command(name = "validate", alias = "check", alias = "verify")]
     Validate {
@@ -173,6 +179,10 @@ async fn main_int() -> anyhow::Result<()> {
         } => {
             apply_patch(src_file, diff_file).await?;
         }
+        Commands::UpdateMetadata { file } => {
+            let mbt = Mbtiles::new(file.as_path())?;
+            mbt.update_metadata().await?;
+        }
         Commands::Validate {
             file,
             integrity_check,

mbtiles/src/lib.rs

@@ -26,6 +26,8 @@ pub use queries::*;
 
 mod summary;
 
+mod update;
+
 mod validation;
 pub use validation::{
     calc_agg_tiles_hash, AggHashType, IntegrityCheckType, MbtType, AGG_TILES_HASH,

mbtiles/src/update.rs

@@ -0,0 +1,33 @@
+use log::info;
+use sqlx::query;
+
+use crate::errors::MbtResult;
+use crate::Mbtiles;
+
+impl Mbtiles {
+    pub async fn update_metadata(&self) -> MbtResult<()> {
+        let mut conn = self.open().await?;
+
+        let info = query!(
+            "
+    SELECT min(zoom_level) AS min_zoom,
+           max(zoom_level) AS max_zoom
+    FROM tiles"
+        )
+        .fetch_one(&mut conn)
+        .await?;
+
+        if let Some(min_zoom) = info.min_zoom {
+            info!("Updating minzoom to {min_zoom}");
+            self.set_metadata_value(&mut conn, "minzoom", &min_zoom)
+                .await?;
+        }
+        if let Some(max_zoom) = info.max_zoom {
+            info!("Updating maxzoom to {max_zoom}");
+            self.set_metadata_value(&mut conn, "maxzoom", &max_zoom)
+                .await?;
+        }
+
+        Ok(())
+    }
+}

mbtiles/tests/copy.rs

@@ -246,6 +246,16 @@ fn databases() -> Databases {
     })
 }
 
+#[actix_rt::test]
+async fn update() -> MbtResult<()> {
+    let (mbt, mut cn) = new_file_no_hash!(databases, Flat, METADATA_V1, TILES_V1, "update");
+    mbt.update_metadata().await?;
+    let dmp = dump(&mut cn).await?;
+    assert_snapshot!(&dmp, "update");
+
+    Ok(())
+}
+
 #[rstest]
 #[trace]
 #[actix_rt::test]

mbtiles/tests/snapshots/copy__update@update.snap

@@ -0,0 +1,51 @@
+---
+source: mbtiles/tests/copy.rs
+expression: actual_value
+---
+[[]]
+type = 'table'
+tbl_name = 'metadata'
+sql = '''
+CREATE TABLE metadata (
+             name text NOT NULL PRIMARY KEY,
+             value text)'''
+values = [
+    '(  "maxzoom", "6"  )',
+    '(  "md-edit", "value - v1"  )',
+    '(  "md-remove", "value - remove"  )',
+    '(  "md-same", "value - same"  )',
+    '(  "minzoom", "3"  )',
+]
+
+[[]]
+type = 'table'
+tbl_name = 'tiles'
+sql = '''
+CREATE TABLE tiles (
+             zoom_level integer NOT NULL,
+             tile_column integer NOT NULL,
+             tile_row integer NOT NULL,
+             tile_data blob,
+             PRIMARY KEY(zoom_level, tile_column, tile_row))'''
+values = [
+    '(  3, 6, 7, blob(root)  )',
+    '(  5, 0, 0, blob(same)  )',
+    '(  5, 0, 1, blob()  )',
+    '(  5, 1, 1, blob(edit-v1)  )',
+    '(  5, 1, 2, blob()  )',
+    '(  5, 1, 3, blob(non-empty)  )',
+    '(  5, 2, 2, blob(remove)  )',
+    '(  5, 2, 3, blob()  )',
+    '(  6, 0, 3, blob(same)  )',
+    '(  6, 0, 5, blob(1-keep-1-rm)  )',
+    '(  6, 1, 4, blob(edit-v1)  )',
+    '(  6, 2, 6, blob(1-keep-1-rm)  )',
+]
+
+[[]]
+type = 'index'
+tbl_name = 'metadata'
+
+[[]]
+type = 'index'
+tbl_name = 'tiles'

tests/expected/martin-cp/flat-with-hash_metadata.txt

@@ -7,8 +7,10 @@ tilejson:
   tilejson: 3.0.0
   tiles: []
   description: public.function_zxy_query_test
+  maxzoom: 6
+  minzoom: 0
   name: function_zxy_query_test
-  format: mvt
+  format: pbf
   generator: martin-cp v0.0.0
 agg_tiles_hash: 9B931A386D6075D1DA55323BD4DBEDAE
 

tests/expected/martin-cp/flat_metadata.txt

@@ -15,9 +15,11 @@ tilejson:
   - -1.0
   - 142.84131509869133
   - 45.0
+  maxzoom: 6
+  minzoom: 0
   name: table_source
   foo: '{"bar":"foo"}'
-  format: mvt
+  format: pbf
   generator: martin-cp v0.0.0
 agg_tiles_hash: EF19FCBCE73ADE1C85E856E6BBA9B4C7