Add metadat copy/apply-diff, new testing framework (#921)

* Fix metadata copying
* Introduce a new metadata field `agg_tiles_hash_after_apply` for diff
files
* Added a lot of new info and debug logging
* Simplified Copying interface - not much value in having all the
complex builder pattern here it seems, might as well use a simple
object.

## Testing
* Generate SQLite DBs in memory on the fly to validate just what we need
* Use `insta` for validating DB content

There is now a function `dump(connection) -> Vec<Entry>` to dump the
content of the entire SQLite DB into text with `serde`. At many steps
through the testing, the DB content is validated with the corresponding
.snap file with `insta` crate (which makes this process mega-simple,
including a simple way to "bless" (update) any changes).

## Discovered bugs
* Seems like normalized files do not get copied properly - they contain
extras that should be removed.

Cargo.toml

@@ -24,10 +24,12 @@ clap = { version = "4", features = ["derive"] }
 criterion = { version = "0.5", features = ["async_futures", "async_tokio", "html_reports"] }
 ctor = "0.2"
 deadpool-postgres = "0.11"
+enum-display = "0.1"
 env_logger = "0.10"
 flate2 = "1"
 futures = "0.3"
 indoc = "2"
+insta = { version = "1", features = ["toml"] }
 itertools = "0.11"
 json-patch = "1.2"
 log = "0.4"
@@ -38,7 +40,9 @@ pmtiles = { version = "0.3", features = ["mmap-async-tokio", "tilejson"] }
 postgis = "0.9"
 postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] }
 postgres-protocol = "0.6"
+pretty_assertions = "1"
 regex = "1"
+rstest = "0.18"
 rustls = { version = "0.21", features = ["dangerous_configuration"] }
 rustls-native-certs = "0.6"
 rustls-pemfile = "1"
@@ -47,7 +51,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
 spreet = { version = "0.9", default-features = false }
-sqlite-hashes = "0.5"
+sqlite-hashes = { version = "0.5", default-features = false, features = ["md5", "window", "hex"] }
 sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio"] }
 subst = { version = "0.3", features = ["yaml"] }
 thiserror = "1"
@@ -58,3 +62,9 @@ tokio-postgres-rustls = "0.10"
 [profile.dev.package]
 # See https://github.com/launchbadge/sqlx#compile-time-verification
 sqlx-macros.opt-level = 3
+# See https://docs.rs/insta/latest/insta/#optional-faster-runs
+insta.opt-level = 3
+similar.opt-level = 3
+
+#[patch.crates-io]
+#sqlite-hashes = { path = "/home/nyurik/dev/rust/sqlite-hashes" }

README.md

@@ -23,11 +23,11 @@ _See [installation instructions](https://maplibre.org/martin/installation.html)
 
 You can download martin from [GitHub releases page](https://github.com/maplibre/martin/releases).
 
-| Platform | AMD-64                                                                                       | ARM-64                              |
-|----------|----------------------------------------------------------------------------------------------|-------------------------------------|
+| Platform | AMD-64                                                                                           | ARM-64                              |
+|----------|--------------------------------------------------------------------------------------------------|-------------------------------------|
 | Linux    | [.tar.gz][rl-linux-x64] (gnu)<br>[.tar.gz][rl-linux-x64-musl] (musl)<br>[.deb][rl-linux-x64-deb] | [.tar.gz][rl-linux-a64-musl] (musl) |
-| macOS    | [.tar.gz][rl-macos-x64]                                                                      | [.tar.gz][rl-macos-a64]             |
-| Windows  | [.zip][rl-win64-zip]                                                                         |                                     |
+| macOS    | [.tar.gz][rl-macos-x64]                                                                          | [.tar.gz][rl-macos-a64]             |
+| Windows  | [.zip][rl-win64-zip]                                                                             |                                     |
 
 [rl-linux-x64]: https://github.com/maplibre/martin/releases/latest/download/martin-Linux-x86_64.tar.gz
 [rl-linux-x64-musl]: https://github.com/maplibre/martin/releases/latest/download/martin-Linux-x86_64-musl.tar.gz

docs/src/development.md

@@ -54,7 +54,7 @@ Available recipes:
     test                   # Run all tests using a test database
     test-ssl               # Run all tests using an SSL connection to a test database. Expected output won't match.
     test-legacy            # Run all tests using the oldest supported version of the database
-    test-unit *ARGS        # Run Rust unit and doc tests (cargo test)
+    test-cargo *ARGS       # Run Rust unit and doc tests (cargo test)
     test-int               # Run integration tests
     bless                  # Run integration tests and save its output as the new expected output
     book                   # Build and open mdbook documentation

docs/src/tools.md

@@ -36,7 +36,10 @@ mbtiles copy src_file.mbtiles dst_file.mbtiles \
         --min-zoom 0 --max-zoom 10
 ```
 
-Copy command can also be used to compare two mbtiles files and generate a diff.
+Copy command can also be used to compare two mbtiles files and generate a delta (diff) file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.  The delta file will contain all tiles that are different between the two files (modifications, insertions, and deletions as `NULL` values), for both the tile and metadata tables.  
+
+There is one exception: `agg_tiles_hash` metadata value will be renamed to `agg_tiles_hash_in_diff`, and a new `agg_tiles_hash` will be generated for the diff file itself. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. The `apply-diff` command will automatically rename the `agg_tiles_hash_in_diff` value back to `agg_tiles_hash` when applying the diff.
+
 ```shell
 mbtiles copy src_file.mbtiles diff_file.mbtiles \
          --diff-with-file modified_file.mbtiles
@@ -49,6 +52,9 @@ mbtiles copy normalized.mbtiles dst.mbtiles \
 ```
 ### apply-diff
 Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.
+
+Note that the `agg_tiles_hash_in_diff` metadata value will be renamed to `agg_tiles_hash` when applying the diff. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied.
+
 ```shell
 mbtiles apply_diff src_file.mbtiles diff_file.mbtiles
 ```
@@ -105,7 +111,7 @@ CREATE VIEW tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM ti
 ```sql, ignore
 CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT);
 CREATE UNIQUE INDEX map_index ON map (zoom_level, tile_column, tile_row);
-CREATE TABLE images (tile_data blob, tile_id text);
+CREATE TABLE images (tile_id text, tile_data blob);
 CREATE UNIQUE INDEX images_id ON images (tile_id);
 CREATE VIEW tiles AS
   SELECT
@@ -127,8 +133,7 @@ CREATE VIEW tiles_with_hash AS
       map.tile_row AS tile_row,
       images.tile_data AS tile_data,
       images.tile_id AS tile_hash
-  FROM map
-  JOIN images ON images.tile_id = map.tile_id;
+  FROM map LEFT JOIN images ON map.tile_id = images.tile_id;
 ```
 
 **__Note:__** All `normalized` files created by the `mbtiles` tool will contain this view.

justfile

@@ -6,8 +6,9 @@ export PGPORT := "5411"
 export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db"
 export CARGO_TERM_COLOR := "always"
 
-# export RUST_LOG := "debug"
-# export RUST_BACKTRACE := "1"
+#export RUST_LOG := "debug"
+#export RUST_LOG := "sqlx::query=info,trace"
+#export RUST_BACKTRACE := "1"
 
 @_default:
     just --list --unsorted
@@ -88,10 +89,10 @@ bench-http: (cargo-install "oha")
     oha -z 120s  http://localhost:3000/function_zxy_query/18/235085/122323
 
 # Run all tests using a test database
-test: start test-unit test-int
+test: start (test-cargo "--all-targets") test-doc test-int
 
 # Run all tests using an SSL connection to a test database. Expected output won't match.
-test-ssl: start-ssl test-unit clean-test
+test-ssl: start-ssl (test-cargo "--all-targets") test-doc clean-test
     tests/test.sh
 
 # Run all tests using an SSL connection with client cert to a test database. Expected output won't match.
@@ -107,16 +108,21 @@ test-ssl-cert: start-ssl-cert
     export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
     export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
     export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key"
-    {{just_executable()}} test-unit clean-test
+    {{just_executable()}} test-cargo --all-targets
+    {{just_executable()}} clean-test
+    {{just_executable()}} test-doc
     tests/test.sh
 
 # Run all tests using the oldest supported version of the database
-test-legacy: start-legacy test-unit test-int
+test-legacy: start-legacy (test-cargo "--all-targets") test-doc test-int
 
-# Run Rust unit and doc tests (cargo test)
-test-unit *ARGS:
-    cargo test --all-targets {{ ARGS }}
-    cargo test --doc
+# Run Rust unit tests (cargo test)
+test-cargo *ARGS:
+    cargo test {{ ARGS }}
+
+# Run Rust doc tests
+test-doc *ARGS:
+    cargo test --doc {{ ARGS }}
 
 # Run integration tests
 test-int: clean-test install-sqlx
@@ -132,13 +138,18 @@ test-int: clean-test install-sqlx
     fi
 
 # Run integration tests and save its output as the new expected output
-bless: start clean-test
+bless: start clean-test bless-insta
     rm -rf tests/temp
     cargo test -p martin --features bless-tests
     tests/test.sh
     rm -rf tests/expected
     mv tests/output tests/expected
 
+# Run integration tests and save its output as the new expected output
+bless-insta *ARGS: (cargo-install "insta" "cargo-insta")
+    #rm -rf martin-mbtiles/tests/snapshots
+    cargo insta test --accept --unreferenced=auto -p martin-mbtiles {{ ARGS }}
+
 # Build and open mdbook documentation
 book: (cargo-install "mdbook")
     mdbook serve docs --open --port 8321

martin-mbtiles/Cargo.toml

@@ -14,6 +14,7 @@ default = ["cli"]
 cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio"]
 
 [dependencies]
+enum-display.workspace = true
 futures.workspace = true
 log.workspace = true
 martin-tile-utils.workspace = true
@@ -34,6 +35,11 @@ tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }
 [dev-dependencies]
 # For testing, might as well use the same async framework as the Martin itself
 actix-rt.workspace = true
+ctor.workspace = true
+env_logger.workspace = true
+insta.workspace = true
+pretty_assertions.workspace = true
+rstest.workspace = true
 
 [lib]
 path = "src/lib.rs"