Add metadat copy/apply-diff, new testing framework

* Generate SQLite DBs in memory on the fly to validate just what we need * Use `insta` for validating all outputs * Fix metadata copying * Introduce a new metadata field `agg_tiles_hash_after_apply` for diff files * Added a lot of new info and debug logging
maplibre · Oct 4, 2023 · 239e057 · 239e057
1 parent 0459d3f
commit 239e057
Show file tree

Hide file tree

Showing 51 changed files with 2,663 additions and 191 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -24,10 +24,12 @@ clap = { version = "4", features = ["derive"] }
 criterion = { version = "0.5", features = ["async_futures", "async_tokio", "html_reports"] }
 ctor = "0.2"
 deadpool-postgres = "0.11"
+enum-display = "0.1"
 env_logger = "0.10"
 flate2 = "1"
 futures = "0.3"
 indoc = "2"
+insta = { version = "1", features = ["yaml", "redactions", "glob", "json", "toml"] }
 itertools = "0.11"
 json-patch = "1.1"
 log = "0.4"
@@ -38,6 +40,7 @@ pmtiles = { version = "0.3", features = ["mmap-async-tokio", "tilejson"] }
 postgis = "0.9"
 postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] }
 postgres-protocol = "0.6"
+pretty_assertions = "1"
 regex = "1"
 rustls = { version = "0.21", features = ["dangerous_configuration"] }
 rustls-native-certs = "0.6"
@@ -58,3 +61,6 @@ tokio-postgres-rustls = "0.10"
 [profile.dev.package]
 # See https://github.com/launchbadge/sqlx#compile-time-verification
 sqlx-macros.opt-level = 3
+# See https://docs.rs/insta/latest/insta/#optional-faster-runs
+insta.opt-level = 3
+similar.opt-level = 3
diff --git a/docs/src/development.md b/docs/src/development.md
@@ -54,7 +54,7 @@ Available recipes:
     test                   # Run all tests using a test database
     test-ssl               # Run all tests using an SSL connection to a test database. Expected output won't match.
     test-legacy            # Run all tests using the oldest supported version of the database
-    test-unit *ARGS        # Run Rust unit and doc tests (cargo test)
+    test-cargo *ARGS       # Run Rust unit and doc tests (cargo test)
     test-int               # Run integration tests
     bless                  # Run integration tests and save its output as the new expected output
     book                   # Build and open mdbook documentation

diff --git a/docs/src/tools.md b/docs/src/tools.md
@@ -36,7 +36,10 @@ mbtiles copy src_file.mbtiles dst_file.mbtiles \
         --min-zoom 0 --max-zoom 10
 ```
 
-Copy command can also be used to compare two mbtiles files and generate a diff.
+Copy command can also be used to compare two mbtiles files and generate a delta (diff) file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.  The delta file will contain all tiles that are different between the two files (modifications, insertions, and deletions as `NULL` values), for both the tile and metadata tables.  
+
+There is one exception: `agg_tiles_hash` metadata value will be renamed to `agg_tiles_hash_in_diff`, and a new `agg_tiles_hash` will be generated for the diff file itself. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. The `apply-diff` command will automatically rename the `agg_tiles_hash_in_diff` value back to `agg_tiles_hash` when applying the diff.
+
 ```shell
 mbtiles copy src_file.mbtiles diff_file.mbtiles \
          --diff-with-file modified_file.mbtiles
@@ -49,6 +52,9 @@ mbtiles copy normalized.mbtiles dst.mbtiles \
 ```
 ### apply-diff
 Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.
+
+Note that the `agg_tiles_hash_in_diff` metadata value will be renamed to `agg_tiles_hash` when applying the diff. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied.
+
 ```shell
 mbtiles apply_diff src_file.mbtiles diff_file.mbtiles
 ```

diff --git a/justfile b/justfile
@@ -6,8 +6,9 @@ export PGPORT := "5411"
 export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db"
 export CARGO_TERM_COLOR := "always"
 
-# export RUST_LOG := "debug"
-# export RUST_BACKTRACE := "1"
+#export RUST_LOG := "debug"
+#export RUST_LOG := "sqlx::query=info,trace"
+#export RUST_BACKTRACE := "1"
 
 @_default:
     just --list --unsorted
@@ -88,10 +89,10 @@ bench-http: (cargo-install "oha")
     oha -z 120s  http://localhost:3000/function_zxy_query/18/235085/122323
 
 # Run all tests using a test database
-test: start test-unit test-int
+test: start (test-cargo "--all-targets") test-doc test-int
 
 # Run all tests using an SSL connection to a test database. Expected output won't match.
-test-ssl: start-ssl test-unit clean-test
+test-ssl: start-ssl (test-cargo "--all-targets") test-doc clean-test
     tests/test.sh
 
 # Run all tests using an SSL connection with client cert to a test database. Expected output won't match.
@@ -107,16 +108,21 @@ test-ssl-cert: start-ssl-cert
     export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
     export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
     export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key"
-    {{just_executable()}} test-unit clean-test
+    {{just_executable()}} test-cargo --all-targets
+    {{just_executable()}} clean-test
+    {{just_executable()}} test-doc
     tests/test.sh
 
 # Run all tests using the oldest supported version of the database
-test-legacy: start-legacy test-unit test-int
+test-legacy: start-legacy (test-cargo "--all-targets") test-doc test-int
 
-# Run Rust unit and doc tests (cargo test)
-test-unit *ARGS:
-    cargo test --all-targets {{ ARGS }}
-    cargo test --doc
+# Run Rust unit tests (cargo test)
+test-cargo *ARGS:
+    cargo test {{ ARGS }}
+
+# Run Rust doc tests
+test-doc *ARGS:
+    cargo test --doc {{ ARGS }}
 
 # Run integration tests
 test-int: clean-test install-sqlx
@@ -132,13 +138,17 @@ test-int: clean-test install-sqlx
     fi
 
 # Run integration tests and save its output as the new expected output
-bless: start clean-test
+bless: start clean-test bless-insta
     rm -rf tests/temp
     cargo test -p martin --features bless-tests
     tests/test.sh
     rm -rf tests/expected
     mv tests/output tests/expected
 
+bless-insta *ARGS: (cargo-install "insta" "cargo-insta")
+    #rm -rf martin-mbtiles/tests/snapshots
+    cargo insta test --accept --unreferenced=auto -p martin-mbtiles {{ ARGS }}
+
 # Build and open mdbook documentation
 book: (cargo-install "mdbook")
     mdbook serve docs --open --port 8321

diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml
@@ -14,6 +14,7 @@ default = ["cli"]
 cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio"]
 
 [dependencies]
+enum-display.workspace = true
 futures.workspace = true
 log.workspace = true
 martin-tile-utils.workspace = true
@@ -34,6 +35,10 @@ tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }
 [dev-dependencies]
 # For testing, might as well use the same async framework as the Martin itself
 actix-rt.workspace = true
+ctor.workspace = true
+env_logger.workspace = true
+insta.workspace = true
+pretty_assertions.workspace = true
 
 [lib]
 path = "src/lib.rs"

diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs
@@ -198,24 +198,25 @@ mod tests {
 
     #[test]
     fn test_copy_min_max_zoom_arguments() {
+        let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"));
+        opt.min_zoom = Some(1);
+        opt.max_zoom = Some(100);
+
+        let args = Args::parse_from([
+            "mbtiles",
+            "copy",
+            "src_file",
+            "dst_file",
+            "--max-zoom",
+            "100",
+            "--min-zoom",
+            "1",
+        ]);
         assert_eq!(
-            Args::parse_from([
-                "mbtiles",
-                "copy",
-                "src_file",
-                "dst_file",
-                "--max-zoom",
-                "100",
-                "--min-zoom",
-                "1"
-            ]),
+            args,
             Args {
                 verbose: false,
-                command: Copy(
-                    MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
-                        .min_zoom(Some(1))
-                        .max_zoom(Some(100))
-                )
+                command: Copy(opt)
             }
         );
     }
@@ -260,6 +261,8 @@ mod tests {
 
     #[test]
     fn test_copy_zoom_levels_arguments() {
+        let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"));
+        opt.zoom_levels.extend(&[1, 3, 7]);
         assert_eq!(
             Args::parse_from([
                 "mbtiles",
@@ -271,16 +274,15 @@ mod tests {
             ]),
             Args {
                 verbose: false,
-                command: Copy(
-                    MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
-                        .zoom_levels(vec![1, 3, 7])
-                )
+                command: Copy(opt)
             }
         );
     }
 
     #[test]
     fn test_copy_diff_with_file_arguments() {
+        let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"));
+        opt.diff_with_file = Some(PathBuf::from("no_file"));
         assert_eq!(
             Args::parse_from([
                 "mbtiles",
@@ -292,16 +294,15 @@ mod tests {
             ]),
             Args {
                 verbose: false,
-                command: Copy(
-                    MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
-                        .diff_with_file(PathBuf::from("no_file"))
-                )
+                command: Copy(opt)
             }
         );
     }
 
     #[test]
     fn test_copy_diff_with_override_copy_duplicate_mode() {
+        let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"));
+        opt.on_duplicate = CopyDuplicateMode::Override;
         assert_eq!(
             Args::parse_from([
                 "mbtiles",
@@ -313,16 +314,15 @@ mod tests {
             ]),
             Args {
                 verbose: false,
-                command: Copy(
-                    MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
-                        .on_duplicate(CopyDuplicateMode::Override)
-                )
+                command: Copy(opt)
             }
         );
     }
 
     #[test]
     fn test_copy_diff_with_ignore_copy_duplicate_mode() {
+        let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"));
+        opt.on_duplicate = CopyDuplicateMode::Ignore;
         assert_eq!(
             Args::parse_from([
                 "mbtiles",
@@ -334,16 +334,15 @@ mod tests {
             ]),
             Args {
                 verbose: false,
-                command: Copy(
-                    MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
-                        .on_duplicate(CopyDuplicateMode::Ignore)
-                )
+                command: Copy(opt)
             }
         );
     }
 
     #[test]
     fn test_copy_diff_with_abort_copy_duplicate_mode() {
+        let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"));
+        opt.on_duplicate = CopyDuplicateMode::Abort;
         assert_eq!(
             Args::parse_from([
                 "mbtiles",
@@ -355,10 +354,7 @@ mod tests {
             ]),
             Args {
                 verbose: false,
-                command: Copy(
-                    MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
-                        .on_duplicate(CopyDuplicateMode::Abort)
-                )
+                command: Copy(opt)
             }
         );
     }