diff --git a/docs/src/martin-cp.md b/docs/src/martin-cp.md index 3d3402026..e3bc7fa9b 100644 --- a/docs/src/martin-cp.md +++ b/docs/src/martin-cp.md @@ -1,6 +1,8 @@ # Generating Tiles in Bulk -`martin-cp` is a tool for generating tiles in bulk, and save retrieved tiles into a new or an existing MBTiles file. It can be used to generate tiles for a large area or multiple areas. If multiple areas overlap, it will generate tiles only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources. +`martin-cp` is a tool for generating tiles in bulk, from any source(s) supported by Martin, and save retrieved tiles into a new or an existing MBTiles file. `martin-cp` can be used to generate tiles for a large area or multiple areas (bounding boxes). If multiple areas overlap, it will ensure each tile is generated only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources. + +After copying, `martin-cp` will update the `agg_tiles_hash` metadata value unless `--skip-agg-tiles-hash` is specified. This allows the MBTiles file to be [validated](./mbtiles-validation.md#aggregate-content-validation) using `mbtiles validate` command. ## Usage @@ -12,6 +14,8 @@ martin-cp --output-file tileset.mbtiles \ "--bbox=-180,-90,180,90" \ --min-zoom 0 \ --max-zoom 10 \ - --source my_table \ + --source source_name \ postgresql://postgres@localhost:5432/db ``` + +You diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index ae51d5001..18d9395cb 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -96,6 +96,9 @@ pub struct CopyArgs { /// List of zoom levels to copy #[arg(short, long, alias = "zooms", value_delimiter = ',')] pub zoom_levels: Vec, + /// Skip generating a global hash for mbtiles validation. By default, `martin-cp` will compute and update `agg_tiles_hash` metadata value. + #[arg(long)] + pub skip_agg_tiles_hash: bool, } async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { @@ -275,9 +278,8 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> let progress = Progress::new(&tiles); info!( - "Copying {} {} tiles from {} to {}", + "Copying {} {tile_info} tiles from {} to {}", progress.total, - tile_info, args.source, args.output_file.display() ); @@ -333,6 +335,16 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> )?; info!("{progress}"); + + if !args.skip_agg_tiles_hash { + if progress.non_empty.load(Ordering::Relaxed) == 0 { + info!("No tiles were copied, skipping agg_tiles_hash computation"); + } else { + info!("Computing agg_tiles_hash value..."); + mbt.update_agg_tiles_hash(&mut conn).await?; + } + } + Ok(()) } diff --git a/mbtiles/src/metadata.rs b/mbtiles/src/metadata.rs index affdc9564..728267442 100644 --- a/mbtiles/src/metadata.rs +++ b/mbtiles/src/metadata.rs @@ -22,6 +22,7 @@ pub struct Metadata { pub layer_type: Option, pub tilejson: TileJSON, pub json: Option, + pub agg_tiles_hash: Option, } #[allow(clippy::trivially_copy_pass_by_ref)] @@ -89,33 +90,16 @@ impl Mbtiles { } pub async fn get_metadata(&self, conn: &mut T) -> MbtResult - where - for<'e> &'e mut T: SqliteExecutor<'e>, - { - let (tj, layer_type, json) = self.parse_metadata(conn).await?; - - Ok(Metadata { - id: self.filename().to_string(), - tile_info: self.detect_format(&tj, conn).await?, - tilejson: tj, - layer_type, - json, - }) - } - - async fn parse_metadata( - &self, - conn: &mut T, - ) -> MbtResult<(TileJSON, Option, Option)> where for<'e> &'e mut T: SqliteExecutor<'e>, { let query = query!("SELECT name, value FROM metadata WHERE value IS NOT ''"); - let mut rows = query.fetch(conn); + let mut rows = query.fetch(&mut *conn); let mut tj = tilejson! { tiles: vec![] }; let mut layer_type: Option = None; let mut json: Option = None; + let mut agg_tiles_hash: Option = None; while let Some(row) = rows.try_next().await? { if let (Some(name), Some(value)) = (row.name, row.value) { @@ -136,6 +120,7 @@ impl Mbtiles { "format" | "generator" => { tj.other.insert(name, Value::String(value)); } + "agg_tiles_hash" => agg_tiles_hash = Some(value), _ => { let file = &self.filename(); info!("{file} has an unrecognized metadata value {name}={value}"); @@ -161,7 +146,17 @@ impl Mbtiles { } } - Ok((tj, layer_type, json)) + // Need to drop rows in order to re-borrow connection reference as mutable + drop(rows); + + Ok(Metadata { + id: self.filename().to_string(), + tile_info: self.detect_format(&tj, &mut *conn).await?, + tilejson: tj, + layer_type, + json, + agg_tiles_hash, + }) } pub async fn insert_metadata(&self, conn: &mut T, tile_json: &TileJSON) -> MbtResult<()> diff --git a/tests/expected/martin-cp/flat-with-hash_metadata.txt b/tests/expected/martin-cp/flat-with-hash_metadata.txt index 71a677635..1be91d17c 100644 --- a/tests/expected/martin-cp/flat-with-hash_metadata.txt +++ b/tests/expected/martin-cp/flat-with-hash_metadata.txt @@ -10,4 +10,5 @@ tilejson: name: function_zxy_query_test format: mvt generator: martin-cp v0.11.2 +agg_tiles_hash: 9B931A386D6075D1DA55323BD4DBEDAE diff --git a/tests/expected/martin-cp/flat_metadata.txt b/tests/expected/martin-cp/flat_metadata.txt index 350447522..f47695741 100644 --- a/tests/expected/martin-cp/flat_metadata.txt +++ b/tests/expected/martin-cp/flat_metadata.txt @@ -19,4 +19,5 @@ tilejson: foo: '{"bar":"foo"}' format: mvt generator: martin-cp v0.11.2 +agg_tiles_hash: EF19FCBCE73ADE1C85E856E6BBA9B4C7 diff --git a/tests/expected/martin-cp/normalized_metadata.txt b/tests/expected/martin-cp/normalized_metadata.txt index ee49f8feb..af7de9ec5 100644 --- a/tests/expected/martin-cp/normalized_metadata.txt +++ b/tests/expected/martin-cp/normalized_metadata.txt @@ -36,4 +36,5 @@ tilejson: version: 1.0.0 format: png generator: martin-cp v0.11.2 +agg_tiles_hash: A85C80BA1CE047E2D93DAC25C5179775