add agg-tiles-hash calc to martin-cp (#1033)

* Automatically computes `agg-tiles-hash` metadata value after copying
tiles
* Add a `--skip-agg-tiles-hash` option

docs/src/martin-cp.md

@@ -1,6 +1,8 @@
 # Generating Tiles in Bulk
 
-`martin-cp` is a tool for generating tiles in bulk, and save retrieved tiles into a new or an existing MBTiles file. It can be used to generate tiles for a large area or multiple areas. If multiple areas overlap, it will generate tiles only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources.
+`martin-cp` is a tool for generating tiles in bulk, from any source(s) supported by Martin, and save retrieved tiles into a new or an existing MBTiles file. `martin-cp` can be used to generate tiles for a large area or multiple areas (bounding boxes). If multiple areas overlap, it will ensure each tile is generated only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources.
+
+After copying, `martin-cp` will update the `agg_tiles_hash` metadata value unless `--skip-agg-tiles-hash` is specified. This allows the MBTiles file to be [validated](./mbtiles-validation.md#aggregate-content-validation) using `mbtiles validate` command.
 
 ## Usage
 
@@ -12,6 +14,8 @@ martin-cp  --output-file tileset.mbtiles \
            "--bbox=-180,-90,180,90"      \
            --min-zoom 0                  \
            --max-zoom 10                 \
-           --source my_table             \
+           --source source_name          \
            postgresql://postgres@localhost:5432/db
 ```
+
+You 

martin/src/bin/martin-cp.rs

@@ -96,6 +96,9 @@ pub struct CopyArgs {
     /// List of zoom levels to copy
     #[arg(short, long, alias = "zooms", value_delimiter = ',')]
     pub zoom_levels: Vec<u8>,
+    /// Skip generating a global hash for mbtiles validation. By default, `martin-cp` will compute and update `agg_tiles_hash` metadata value.
+    #[arg(long)]
+    pub skip_agg_tiles_hash: bool,
 }
 
 async fn start(copy_args: CopierArgs) -> MartinCpResult<()> {
@@ -275,9 +278,8 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()>
 
     let progress = Progress::new(&tiles);
     info!(
-        "Copying {} {} tiles from {} to {}",
+        "Copying {} {tile_info} tiles from {} to {}",
         progress.total,
-        tile_info,
         args.source,
         args.output_file.display()
     );
@@ -333,6 +335,16 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()>
     )?;
 
     info!("{progress}");
+
+    if !args.skip_agg_tiles_hash {
+        if progress.non_empty.load(Ordering::Relaxed) == 0 {
+            info!("No tiles were copied, skipping agg_tiles_hash computation");
+        } else {
+            info!("Computing agg_tiles_hash value...");
+            mbt.update_agg_tiles_hash(&mut conn).await?;
+        }
+    }
+
     Ok(())
 }
 

mbtiles/src/metadata.rs

@@ -22,6 +22,7 @@ pub struct Metadata {
     pub layer_type: Option<String>,
     pub tilejson: TileJSON,
     pub json: Option<JSONValue>,
+    pub agg_tiles_hash: Option<String>,
 }
 
 #[allow(clippy::trivially_copy_pass_by_ref)]
@@ -89,33 +90,16 @@ impl Mbtiles {
     }
 
     pub async fn get_metadata<T>(&self, conn: &mut T) -> MbtResult<Metadata>
-    where
-        for<'e> &'e mut T: SqliteExecutor<'e>,
-    {
-        let (tj, layer_type, json) = self.parse_metadata(conn).await?;
-
-        Ok(Metadata {
-            id: self.filename().to_string(),
-            tile_info: self.detect_format(&tj, conn).await?,
-            tilejson: tj,
-            layer_type,
-            json,
-        })
-    }
-
-    async fn parse_metadata<T>(
-        &self,
-        conn: &mut T,
-    ) -> MbtResult<(TileJSON, Option<String>, Option<Value>)>
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
         let query = query!("SELECT name, value FROM metadata WHERE value IS NOT ''");
-        let mut rows = query.fetch(conn);
+        let mut rows = query.fetch(&mut *conn);
 
         let mut tj = tilejson! { tiles: vec![] };
         let mut layer_type: Option<String> = None;
         let mut json: Option<JSONValue> = None;
+        let mut agg_tiles_hash: Option<String> = None;
 
         while let Some(row) = rows.try_next().await? {
             if let (Some(name), Some(value)) = (row.name, row.value) {
@@ -136,6 +120,7 @@ impl Mbtiles {
                     "format" | "generator" => {
                         tj.other.insert(name, Value::String(value));
                     }
+                    "agg_tiles_hash" => agg_tiles_hash = Some(value),
                     _ => {
                         let file = &self.filename();
                         info!("{file} has an unrecognized metadata value {name}={value}");
@@ -161,7 +146,17 @@ impl Mbtiles {
             }
         }
 
-        Ok((tj, layer_type, json))
+        // Need to drop rows in order to re-borrow connection reference as mutable
+        drop(rows);
+
+        Ok(Metadata {
+            id: self.filename().to_string(),
+            tile_info: self.detect_format(&tj, &mut *conn).await?,
+            tilejson: tj,
+            layer_type,
+            json,
+            agg_tiles_hash,
+        })
     }
 
     pub async fn insert_metadata<T>(&self, conn: &mut T, tile_json: &TileJSON) -> MbtResult<()>

tests/expected/martin-cp/flat-with-hash_metadata.txt

@@ -10,4 +10,5 @@ tilejson:
   name: function_zxy_query_test
   format: mvt
   generator: martin-cp v0.11.2
+agg_tiles_hash: 9B931A386D6075D1DA55323BD4DBEDAE
 

tests/expected/martin-cp/flat_metadata.txt

@@ -19,4 +19,5 @@ tilejson:
   foo: '{"bar":"foo"}'
   format: mvt
   generator: martin-cp v0.11.2
+agg_tiles_hash: EF19FCBCE73ADE1C85E856E6BBA9B4C7
 

tests/expected/martin-cp/normalized_metadata.txt

@@ -36,4 +36,5 @@ tilejson:
   version: 1.0.0
   format: png
   generator: martin-cp v0.11.2
+agg_tiles_hash: A85C80BA1CE047E2D93DAC25C5179775