diff --git a/Cargo.lock b/Cargo.lock index cd358dae1..609589a9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,7 +331,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -341,7 +341,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -664,6 +664,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "const-oid" version = "0.9.5" @@ -676,6 +688,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -915,13 +936,19 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", "syn 1.0.109", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "difflib" version = "0.4.0" @@ -961,6 +988,12 @@ dependencies = [ "serde", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -970,6 +1003,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-display" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d4df33d54dd1959d177a0e2c2f4e5a8637a3054aa56861ed7e173ad2043fe2" +dependencies = [ + "enum-display-macro", +] + +[[package]] +name = "enum-display-macro" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ce3a36047ede676eb0d2721d065beed8410cf4f113f489604d2971331cb378" +dependencies = [ + "convert_case 0.6.0", + "quote", + "syn 1.0.109", +] + [[package]] name = "enum_dispatch" version = "0.3.12" @@ -1009,7 +1062,7 @@ checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1030,7 +1083,7 @@ checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ "cfg-if", "home", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1081,7 +1134,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1181,7 +1234,7 @@ dependencies = [ "async-trait", "rustix", "tokio", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1452,7 +1505,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1558,6 +1611,25 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +[[package]] +name = "insta" +version = "1.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa511b2e298cd49b1856746f6bb73e17036bcd66b25f5e92cdcdbec9bd75686" +dependencies = [ + "console", + "globset", + "lazy_static", + "linked-hash-map", + "pest", + "pest_derive", + "serde", + "similar", + "toml", + "walkdir", + "yaml-rust", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -1566,7 +1638,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1694,6 +1766,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.4.8" @@ -1786,10 +1864,14 @@ dependencies = [ "actix-rt", "anyhow", "clap", + "ctor", + "enum-display", "env_logger", "futures", + "insta", "log", "martin-tile-utils", + "pretty_assertions", "serde", "serde_json", "serde_yaml", @@ -1878,7 +1960,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2037,7 +2119,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -2087,6 +2169,51 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pest_meta" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "phf" version = "0.11.2" @@ -2303,6 +2430,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro2" version = "1.0.67" @@ -2538,7 +2675,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2621,7 +2758,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2794,6 +2931,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "similar" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" + [[package]] name = "simplecss" version = "0.2.1" @@ -2840,7 +2983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3220,7 +3363,7 @@ dependencies = [ "fastrand", "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3374,7 +3517,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3463,6 +3606,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "tracing" version = "0.1.37" @@ -3529,6 +3681,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -3839,13 +3997,37 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -3854,51 +4036,93 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3926,6 +4150,21 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zeroize" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 3f806e0b6..17d77db80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,10 +24,12 @@ clap = { version = "4", features = ["derive"] } criterion = { version = "0.5", features = ["async_futures", "async_tokio", "html_reports"] } ctor = "0.2" deadpool-postgres = "0.11" +enum-display = "0.1" env_logger = "0.10" flate2 = "1" futures = "0.3" indoc = "2" +insta = { version = "1", features = ["yaml", "redactions", "glob", "json", "toml"] } itertools = "0.11" json-patch = "1.1" log = "0.4" @@ -38,6 +40,7 @@ pmtiles = { version = "0.3", features = ["mmap-async-tokio", "tilejson"] } postgis = "0.9" postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] } postgres-protocol = "0.6" +pretty_assertions = "1" regex = "1" rustls = { version = "0.21", features = ["dangerous_configuration"] } rustls-native-certs = "0.6" @@ -58,3 +61,6 @@ tokio-postgres-rustls = "0.10" [profile.dev.package] # See https://github.com/launchbadge/sqlx#compile-time-verification sqlx-macros.opt-level = 3 +# See https://docs.rs/insta/latest/insta/#optional-faster-runs +insta.opt-level = 3 +similar.opt-level = 3 diff --git a/docs/src/development.md b/docs/src/development.md index 86891fbed..1475e6c76 100644 --- a/docs/src/development.md +++ b/docs/src/development.md @@ -54,7 +54,7 @@ Available recipes: test # Run all tests using a test database test-ssl # Run all tests using an SSL connection to a test database. Expected output won't match. test-legacy # Run all tests using the oldest supported version of the database - test-unit *ARGS # Run Rust unit and doc tests (cargo test) + test-cargo *ARGS # Run Rust unit and doc tests (cargo test) test-int # Run integration tests bless # Run integration tests and save its output as the new expected output book # Build and open mdbook documentation diff --git a/docs/src/tools.md b/docs/src/tools.md index bbb63082c..1144f31c3 100644 --- a/docs/src/tools.md +++ b/docs/src/tools.md @@ -36,7 +36,10 @@ mbtiles copy src_file.mbtiles dst_file.mbtiles \ --min-zoom 0 --max-zoom 10 ``` -Copy command can also be used to compare two mbtiles files and generate a diff. +Copy command can also be used to compare two mbtiles files and generate a delta (diff) file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset. The delta file will contain all tiles that are different between the two files (modifications, insertions, and deletions as `NULL` values), for both the tile and metadata tables. + +There is one exception: `agg_tiles_hash` metadata value will be renamed to `agg_tiles_hash_in_diff`, and a new `agg_tiles_hash` will be generated for the diff file itself. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. The `apply-diff` command will automatically rename the `agg_tiles_hash_in_diff` value back to `agg_tiles_hash` when applying the diff. + ```shell mbtiles copy src_file.mbtiles diff_file.mbtiles \ --diff-with-file modified_file.mbtiles @@ -49,6 +52,9 @@ mbtiles copy normalized.mbtiles dst.mbtiles \ ``` ### apply-diff Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset. + +Note that the `agg_tiles_hash_in_diff` metadata value will be renamed to `agg_tiles_hash` when applying the diff. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. + ```shell mbtiles apply_diff src_file.mbtiles diff_file.mbtiles ``` diff --git a/justfile b/justfile index b60808eb2..25901a101 100644 --- a/justfile +++ b/justfile @@ -6,8 +6,9 @@ export PGPORT := "5411" export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db" export CARGO_TERM_COLOR := "always" -# export RUST_LOG := "debug" -# export RUST_BACKTRACE := "1" +#export RUST_LOG := "debug" +#export RUST_LOG := "sqlx::query=info,trace" +#export RUST_BACKTRACE := "1" @_default: just --list --unsorted @@ -88,10 +89,10 @@ bench-http: (cargo-install "oha") oha -z 120s http://localhost:3000/function_zxy_query/18/235085/122323 # Run all tests using a test database -test: start test-unit test-int +test: start (test-cargo "--all-targets") test-doc test-int # Run all tests using an SSL connection to a test database. Expected output won't match. -test-ssl: start-ssl test-unit clean-test +test-ssl: start-ssl (test-cargo "--all-targets") test-doc clean-test tests/test.sh # Run all tests using an SSL connection with client cert to a test database. Expected output won't match. @@ -107,16 +108,21 @@ test-ssl-cert: start-ssl-cert export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem" export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem" export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key" - {{just_executable()}} test-unit clean-test + {{just_executable()}} test-cargo --all-targets + {{just_executable()}} clean-test + {{just_executable()}} test-doc tests/test.sh # Run all tests using the oldest supported version of the database -test-legacy: start-legacy test-unit test-int +test-legacy: start-legacy (test-cargo "--all-targets") test-doc test-int -# Run Rust unit and doc tests (cargo test) -test-unit *ARGS: - cargo test --all-targets {{ ARGS }} - cargo test --doc +# Run Rust unit tests (cargo test) +test-cargo *ARGS: + cargo test {{ ARGS }} + +# Run Rust doc tests +test-doc *ARGS: + cargo test --doc {{ ARGS }} # Run integration tests test-int: clean-test install-sqlx @@ -132,13 +138,17 @@ test-int: clean-test install-sqlx fi # Run integration tests and save its output as the new expected output -bless: start clean-test +bless: start clean-test bless-insta rm -rf tests/temp cargo test -p martin --features bless-tests tests/test.sh rm -rf tests/expected mv tests/output tests/expected +bless-insta *ARGS: (cargo-install "insta" "cargo-insta") + #rm -rf martin-mbtiles/tests/snapshots + cargo insta test --accept --unreferenced=auto -p martin-mbtiles {{ ARGS }} + # Build and open mdbook documentation book: (cargo-install "mdbook") mdbook serve docs --open --port 8321 diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index 155792850..7d5f12003 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -14,6 +14,7 @@ default = ["cli"] cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio"] [dependencies] +enum-display.workspace = true futures.workspace = true log.workspace = true martin-tile-utils.workspace = true @@ -34,6 +35,10 @@ tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } [dev-dependencies] # For testing, might as well use the same async framework as the Martin itself actix-rt.workspace = true +ctor.workspace = true +env_logger.workspace = true +insta.workspace = true +pretty_assertions.workspace = true [lib] path = "src/lib.rs" diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index a555794d0..a1aa174d5 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -198,24 +198,25 @@ mod tests { #[test] fn test_copy_min_max_zoom_arguments() { + let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); + opt.min_zoom = Some(1); + opt.max_zoom = Some(100); + + let args = Args::parse_from([ + "mbtiles", + "copy", + "src_file", + "dst_file", + "--max-zoom", + "100", + "--min-zoom", + "1", + ]); assert_eq!( - Args::parse_from([ - "mbtiles", - "copy", - "src_file", - "dst_file", - "--max-zoom", - "100", - "--min-zoom", - "1" - ]), + args, Args { verbose: false, - command: Copy( - MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) - .min_zoom(Some(1)) - .max_zoom(Some(100)) - ) + command: Copy(opt) } ); } @@ -260,6 +261,8 @@ mod tests { #[test] fn test_copy_zoom_levels_arguments() { + let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); + opt.zoom_levels.extend(&[1, 3, 7]); assert_eq!( Args::parse_from([ "mbtiles", @@ -271,16 +274,15 @@ mod tests { ]), Args { verbose: false, - command: Copy( - MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) - .zoom_levels(vec![1, 3, 7]) - ) + command: Copy(opt) } ); } #[test] fn test_copy_diff_with_file_arguments() { + let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); + opt.diff_with_file = Some(PathBuf::from("no_file")); assert_eq!( Args::parse_from([ "mbtiles", @@ -292,16 +294,15 @@ mod tests { ]), Args { verbose: false, - command: Copy( - MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) - .diff_with_file(PathBuf::from("no_file")) - ) + command: Copy(opt) } ); } #[test] fn test_copy_diff_with_override_copy_duplicate_mode() { + let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); + opt.on_duplicate = CopyDuplicateMode::Override; assert_eq!( Args::parse_from([ "mbtiles", @@ -313,16 +314,15 @@ mod tests { ]), Args { verbose: false, - command: Copy( - MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) - .on_duplicate(CopyDuplicateMode::Override) - ) + command: Copy(opt) } ); } #[test] fn test_copy_diff_with_ignore_copy_duplicate_mode() { + let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); + opt.on_duplicate = CopyDuplicateMode::Ignore; assert_eq!( Args::parse_from([ "mbtiles", @@ -334,16 +334,15 @@ mod tests { ]), Args { verbose: false, - command: Copy( - MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) - .on_duplicate(CopyDuplicateMode::Ignore) - ) + command: Copy(opt) } ); } #[test] fn test_copy_diff_with_abort_copy_duplicate_mode() { + let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); + opt.on_duplicate = CopyDuplicateMode::Abort; assert_eq!( Args::parse_from([ "mbtiles", @@ -355,10 +354,7 @@ mod tests { ]), Args { verbose: false, - command: Copy( - MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) - .on_duplicate(CopyDuplicateMode::Abort) - ) + command: Copy(opt) } ); } diff --git a/martin-mbtiles/src/copier.rs b/martin-mbtiles/src/copier.rs index ba1a6a9bd..7869625aa 100644 --- a/martin-mbtiles/src/copier.rs +++ b/martin-mbtiles/src/copier.rs @@ -3,21 +3,23 @@ use std::path::PathBuf; #[cfg(feature = "cli")] use clap::{builder::ValueParser, error::ErrorKind, Args, ValueEnum}; +use enum_display::EnumDisplay; +use log::{debug, info}; use sqlite_hashes::rusqlite; use sqlite_hashes::rusqlite::params_from_iter; -use sqlx::sqlite::SqliteConnectOptions; -use sqlx::{query, Connection, Executor as _, Row, SqliteConnection}; +use sqlx::{query, Executor as _, Row, SqliteConnection}; use crate::errors::MbtResult; use crate::mbtiles::MbtType::{Flat, FlatWithHash, Normalized}; -use crate::mbtiles::{attach_hash_fn, MbtType}; +use crate::mbtiles::{detach_db, MbtType}; use crate::queries::{ create_flat_tables, create_flat_with_hash_tables, create_normalized_tables, create_tiles_with_hash_view, }; -use crate::{MbtError, Mbtiles}; +use crate::{MbtError, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF}; -#[derive(PartialEq, Eq, Default, Debug, Clone)] +#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] +#[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(ValueEnum))] pub enum CopyDuplicateMode { #[default] @@ -30,30 +32,31 @@ pub enum CopyDuplicateMode { #[cfg_attr(feature = "cli", derive(Args))] pub struct MbtilesCopier { /// MBTiles file to read from - src_file: PathBuf, + pub src_file: PathBuf, /// MBTiles file to write to - dst_file: PathBuf, + pub dst_file: PathBuf, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source #[cfg_attr(feature = "cli", arg(long, value_enum))] - dst_type: Option, + pub dst_type: Option, /// Specify copying behaviour when tiles with duplicate (zoom_level, tile_column, tile_row) values are found #[cfg_attr(feature = "cli", arg(long, value_enum, default_value_t = CopyDuplicateMode::default()))] - on_duplicate: CopyDuplicateMode, + pub on_duplicate: CopyDuplicateMode, /// Minimum zoom level to copy #[cfg_attr(feature = "cli", arg(long, conflicts_with("zoom_levels")))] - min_zoom: Option, + pub min_zoom: Option, /// Maximum zoom level to copy #[cfg_attr(feature = "cli", arg(long, conflicts_with("zoom_levels")))] - max_zoom: Option, + pub max_zoom: Option, /// List of zoom levels to copy #[cfg_attr(feature = "cli", arg(long, value_parser(ValueParser::new(HashSetValueParser{})), default_value=""))] - zoom_levels: HashSet, - /// Compare source file with this file, and only copy non-identical tiles to destination + pub zoom_levels: HashSet, + /// Compare source file with this file, and only copy non-identical tiles to destination. + /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[cfg_attr(feature = "cli", arg(long))] - diff_with_file: Option, + pub diff_with_file: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. #[cfg_attr(feature = "cli", arg(long))] - skip_agg_tiles_hash: bool, + pub skip_agg_tiles_hash: bool, } #[cfg(feature = "cli")] @@ -111,48 +114,6 @@ impl MbtilesCopier { } } - #[must_use] - pub fn dst_type(mut self, dst_type: Option) -> Self { - self.dst_type = dst_type; - self - } - - #[must_use] - pub fn on_duplicate(mut self, on_duplicate: CopyDuplicateMode) -> Self { - self.on_duplicate = on_duplicate; - self - } - - #[must_use] - pub fn zoom_levels(mut self, zoom_levels: Vec) -> Self { - self.zoom_levels.extend(zoom_levels); - self - } - - #[must_use] - pub fn min_zoom(mut self, min_zoom: Option) -> Self { - self.min_zoom = min_zoom; - self - } - - #[must_use] - pub fn max_zoom(mut self, max_zoom: Option) -> Self { - self.max_zoom = max_zoom; - self - } - - #[must_use] - pub fn diff_with_file(mut self, diff_with_file: PathBuf) -> Self { - self.diff_with_file = Some(diff_with_file); - self - } - - #[must_use] - pub fn skip_agg_tiles_hash(mut self, skip_global_hash: bool) -> Self { - self.skip_agg_tiles_hash = skip_global_hash; - self - } - pub async fn run(self) -> MbtResult { MbtileCopierInt::new(self)?.run().await } @@ -160,6 +121,15 @@ impl MbtilesCopier { impl MbtileCopierInt { pub fn new(options: MbtilesCopier) -> MbtResult { + // We may want to resolve the files to absolute paths here, but will need to avoid various non-file cases + if options.src_file == options.dst_file { + return Err(MbtError::SameSourceAndDestination(options.src_file)); + } + if let Some(diff_file) = &options.diff_with_file { + if options.src_file == *diff_file || options.dst_file == *diff_file { + return Err(MbtError::SameDiffAndSourceOrDestination(options.src_file)); + } + } Ok(MbtileCopierInt { src_mbtiles: Mbtiles::new(&options.src_file)?, dst_mbtiles: Mbtiles::new(&options.dst_file)?, @@ -170,15 +140,7 @@ impl MbtileCopierInt { pub async fn run(self) -> MbtResult { // src file connection is not needed after this point, as it will be attached to the dst file let src_type = self.src_mbtiles.open_and_detect_type().await?; - - let mut conn = SqliteConnection::connect_with( - &SqliteConnectOptions::new() - .create_if_missing(true) - .filename(&self.options.dst_file), - ) - .await?; - - attach_hash_fn(&mut conn).await?; + let mut conn = self.dst_mbtiles.open_or_new().await?; let is_empty = query!("SELECT 1 as has_rows FROM sqlite_schema LIMIT 1") .fetch_optional(&mut conn) @@ -187,12 +149,22 @@ impl MbtileCopierInt { let dst_type = if is_empty { let dst_type = self.options.dst_type.unwrap_or(src_type); - self.copy_to_new(&mut conn, src_type, dst_type).await?; + info!( + "Copying {} ({src_type}) to a new file {} ({dst_type})", + self.options.src_file.display(), + self.options.dst_file.display() + ); + self.init_new_schema(&mut conn, src_type, dst_type).await?; dst_type } else if self.options.diff_with_file.is_some() { return Err(MbtError::NonEmptyTargetFile(self.options.dst_file)); } else { let dst_type = self.dst_mbtiles.detect_type(&mut conn).await?; + info!( + "Copying {} ({src_type}) to an existing file {} ({dst_type})", + self.options.src_file.display(), + self.options.dst_file.display() + ); self.src_mbtiles.attach_to(&mut conn, "sourceDb").await?; dst_type }; @@ -200,11 +172,15 @@ impl MbtileCopierInt { let (on_dupl, sql_cond) = self.get_on_duplicate_sql(dst_type); let (select_from, query_args) = { - let select_from = if let Some(diff_file) = &self.options.diff_with_file { - let diff_with_mbtiles = Mbtiles::new(diff_file)?; - let diff_type = diff_with_mbtiles.open_and_detect_type().await?; - diff_with_mbtiles.attach_to(&mut conn, "newDb").await?; - Self::get_select_from_with_diff(dst_type, diff_type) + let select_from = if let Some(dif_file) = &self.options.diff_with_file { + let dif_file = Mbtiles::new(dif_file)?; + let dif_type = dif_file.open_and_detect_type().await?; + info!( + "Copying only the data not found in {} ({dif_type})", + dif_file.filepath(), + ); + dif_file.attach_to(&mut conn, "diffDb").await?; + Self::get_select_from_with_diff(dst_type, dif_type) } else { Self::get_select_from(dst_type, src_type).to_string() }; @@ -215,6 +191,7 @@ impl MbtileCopierInt { }; { + debug!("Copying tiles with 'INSERT {on_dupl}' {src_type} -> {dst_type} ({sql_cond})"); // Make sure not to execute any other queries while the handle is locked let mut handle_lock = conn.lock_handle().await?; let handle = handle_lock.as_raw_handle().as_ptr(); @@ -247,21 +224,51 @@ impl MbtileCopierInt { )? } }; + + let sql = if self.options.diff_with_file.is_some() { + debug!("Copying metadata with 'INSERT {on_dupl}', taking into account diff file"); + // Insert all rows from diffDb.metadata if they do not exist or are different in sourceDb.metadata. + // Also insert all names from sourceDb.metadata that do not exist in diffDb.metadata, with their value set to NULL. + // Rename agg_tiles_hash to agg_tiles_hash_in_diff because agg_tiles_hash will be auto-added later + format!( + "INSERT {on_dupl} INTO metadata (name, value) + SELECT CASE WHEN dif.name = '{AGG_TILES_HASH}' THEN '{AGG_TILES_HASH_IN_DIFF}' ELSE dif.name END as name, + dif.value as value + FROM diffDb.metadata AS dif LEFT JOIN sourceDb.metadata AS src + ON dif.name = src.name + WHERE (dif.value != src.value OR src.value ISNULL) + AND dif.name != '{AGG_TILES_HASH_IN_DIFF}' + UNION ALL + SELECT src.name as name, NULL as value + FROM sourceDb.metadata AS src LEFT JOIN diffDb.metadata AS dif + ON src.name = dif.name + WHERE dif.value ISNULL AND src.name NOT IN ('{AGG_TILES_HASH}', '{AGG_TILES_HASH_IN_DIFF}');" + ) + } else { + debug!("Copying metadata with 'INSERT {on_dupl}'"); + format!("INSERT {on_dupl} INTO metadata SELECT name, value FROM sourceDb.metadata") + }; + rusqlite_conn.execute(&sql, [])?; } if !self.options.skip_agg_tiles_hash { self.dst_mbtiles.update_agg_tiles_hash(&mut conn).await?; } + detach_db(&mut conn, "sourceDb").await?; + // Ignore error because we might not have attached diffDb + let _ = detach_db(&mut conn, "diffDb").await; + Ok(conn) } - async fn copy_to_new( + async fn init_new_schema( &self, conn: &mut SqliteConnection, src: MbtType, dst: MbtType, ) -> MbtResult<()> { + debug!("Resetting PRAGMA settings and vacuuming"); query!("PRAGMA page_size = 512").execute(&mut *conn).await?; query!("PRAGMA encoding = 'UTF-8'") .execute(&mut *conn) @@ -272,12 +279,13 @@ impl MbtileCopierInt { if src == dst { // DB objects must be created in a specific order: tables, views, triggers, indexes. + debug!("Copying DB schema verbatim"); let sql_objects = conn .fetch_all( "SELECT sql FROM sourceDb.sqlite_schema WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images', 'tiles_with_hash') - AND type IN ('table', 'view', 'trigger', 'index') + AND type IN ('table', 'view', 'trigger', 'index') ORDER BY CASE WHEN type = 'table' THEN 1 WHEN type = 'view' THEN 2 @@ -303,9 +311,6 @@ impl MbtileCopierInt { create_tiles_with_hash_view(&mut *conn).await?; } - conn.execute("INSERT INTO metadata SELECT * FROM sourceDb.metadata") - .await?; - Ok(()) } @@ -338,14 +343,14 @@ impl MbtileCopierInt { fn get_select_from_with_diff(dst_type: MbtType, diff_type: MbtType) -> String { let (hash_col_sql, new_tiles_with_hash) = if dst_type == Flat { - ("", "newDb.tiles") + ("", "diffDb.tiles") } else { match diff_type { - Flat => (", hex(md5(tile_data)) as hash", "newDb.tiles"), - FlatWithHash => (", new_tiles_with_hash.tile_hash as hash", "newDb.tiles_with_hash"), + Flat => (", hex(md5(tile_data)) as hash", "diffDb.tiles"), + FlatWithHash => (", new_tiles_with_hash.tile_hash as hash", "diffDb.tiles_with_hash"), Normalized => (", new_tiles_with_hash.hash", "(SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash - FROM newDb.map JOIN newDb.images ON newDb.map.tile_id = newDb.images.tile_id)"), + FROM diffDb.map JOIN diffDb.images ON diffDb.map.tile_id = diffDb.images.tile_id)"), } }; @@ -453,7 +458,27 @@ pub async fn apply_diff(src_file: PathBuf, diff_file: PathBuf) -> MbtResult<()> .execute(&mut conn) .await?; - Ok(()) + // Copy metadata from diffDb to the destination file, replacing existing values + // Convert 'agg_tiles_hash_in_diff' into 'agg_tiles_hash' + // Delete metadata entries if the value is NULL in diffDb + query(&format!( + "INSERT OR REPLACE INTO metadata (name, value) + SELECT CASE WHEN name = '{AGG_TILES_HASH_IN_DIFF}' THEN '{AGG_TILES_HASH}' ELSE name END as name, + value + FROM diffDb.metadata + WHERE name NOTNULL AND name != '{AGG_TILES_HASH}';" + )) + .execute(&mut conn) + .await?; + + query( + "DELETE FROM metadata + WHERE name IN (SELECT name FROM diffDb.metadata WHERE value ISNULL);", + ) + .execute(&mut conn) + .await?; + + detach_db(&mut conn, "diffDb").await } #[cfg(test)] @@ -475,10 +500,9 @@ mod tests { dst_type: Option, expected_dst_type: MbtType, ) -> MbtResult<()> { - let mut dst_conn = MbtilesCopier::new(src_filepath.clone(), dst_filepath.clone()) - .dst_type(dst_type) - .run() - .await?; + let mut opt = MbtilesCopier::new(src_filepath.clone(), dst_filepath.clone()); + opt.dst_type = dst_type; + let mut dst_conn = opt.run().await?; Mbtiles::new(src_filepath)? .attach_to(&mut dst_conn, "srcDb") @@ -500,10 +524,10 @@ mod tests { } async fn verify_copy_with_zoom_filter( - opts: MbtilesCopier, + opt: MbtilesCopier, expected_zoom_levels: u8, ) -> MbtResult<()> { - let mut dst_conn = opts.run().await?; + let mut dst_conn = opt.run().await?; assert_eq!( get_one::( @@ -594,9 +618,9 @@ mod tests { async fn copy_with_min_max_zoom() -> MbtResult<()> { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); - let opt = MbtilesCopier::new(src, dst) - .min_zoom(Some(2)) - .max_zoom(Some(4)); + let mut opt = MbtilesCopier::new(src, dst); + opt.min_zoom = Some(2); + opt.max_zoom = Some(4); verify_copy_with_zoom_filter(opt, 3).await } @@ -604,10 +628,10 @@ mod tests { async fn copy_with_zoom_levels() -> MbtResult<()> { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); - let opt = MbtilesCopier::new(src, dst) - .min_zoom(Some(2)) - .max_zoom(Some(4)) - .zoom_levels(vec![1, 6]); + let mut opt = MbtilesCopier::new(src, dst); + opt.min_zoom = Some(2); + opt.max_zoom = Some(4); + opt.zoom_levels.extend(&[1, 6]); verify_copy_with_zoom_filter(opt, 2).await } @@ -619,10 +643,9 @@ mod tests { let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"); - let copy_opts = - MbtilesCopier::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone()); - - let mut dst_conn = copy_opts.run().await?; + let mut opt = MbtilesCopier::new(src.clone(), dst.clone()); + opt.diff_with_file = Some(diff_file.clone()); + let mut dst_conn = opt.run().await?; assert!(dst_conn .fetch_optional("SELECT 1 FROM sqlite_schema WHERE name = 'tiles';") @@ -680,11 +703,11 @@ mod tests { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let copy_opts = - MbtilesCopier::new(src.clone(), dst.clone()).on_duplicate(CopyDuplicateMode::Abort); + let mut opt = MbtilesCopier::new(src.clone(), dst.clone()); + opt.on_duplicate = CopyDuplicateMode::Abort; assert!(matches!( - copy_opts.run().await.unwrap_err(), + opt.run().await.unwrap_err(), MbtError::RusqliteError(..) )); } @@ -731,10 +754,9 @@ mod tests { .run() .await?; - let mut dst_conn = MbtilesCopier::new(src_file.clone(), dst.clone()) - .on_duplicate(CopyDuplicateMode::Ignore) - .run() - .await?; + let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone()); + opt.on_duplicate = CopyDuplicateMode::Ignore; + let mut dst_conn = opt.run().await?; // Verify the tiles in the destination file are the same as those in the source file except for those with duplicate (zoom_level, tile_column, tile_row) Mbtiles::new(src_file)? diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs index 43d200037..17501a5fc 100644 --- a/martin-mbtiles/src/errors.rs +++ b/martin-mbtiles/src/errors.rs @@ -5,6 +5,12 @@ use sqlite_hashes::rusqlite; #[derive(thiserror::Error, Debug)] pub enum MbtError { + #[error("The source and destination MBTiles files are the same: {}", .0.display())] + SameSourceAndDestination(PathBuf), + + #[error("The diff file and source or destination MBTiles files are the same: {}", .0.display())] + SameDiffAndSourceOrDestination(PathBuf), + #[error("SQL Error {0}")] SqlxError(#[from] sqlx::Error), diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index dff5b1052..3f8b8b511 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -4,7 +4,9 @@ mod errors; pub use errors::{MbtError, MbtResult}; mod mbtiles; -pub use mbtiles::{IntegrityCheckType, Mbtiles, Metadata}; +pub use mbtiles::{ + IntegrityCheckType, MbtType, Mbtiles, Metadata, AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF, +}; mod pool; pub use pool::MbtilesPool; @@ -13,3 +15,7 @@ mod copier; pub use copier::{apply_diff, CopyDuplicateMode, MbtilesCopier}; mod queries; +pub use queries::{ + create_flat_tables, create_flat_with_hash_tables, create_metadata_table, + create_normalized_tables, is_flat_with_hash_tables_type, is_normalized_tables_type, +}; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index 6cedee852..dc3df79bb 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -8,6 +8,7 @@ use std::str::FromStr; #[cfg(feature = "cli")] use clap::ValueEnum; +use enum_display::EnumDisplay; use futures::TryStreamExt; use log::{debug, info, warn}; use martin_tile_utils::{Format, TileInfo}; @@ -54,7 +55,15 @@ where s.end() } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Metadata key for the aggregate tiles hash value +pub const AGG_TILES_HASH: &str = "agg_tiles_hash"; + +/// Metadata key for a diff file, +/// describing the eventual AGG_TILES_HASH value once the diff is applied +pub const AGG_TILES_HASH_IN_DIFF: &str = "agg_tiles_hash_after_apply"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)] +#[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(ValueEnum))] pub enum MbtType { Flat, @@ -62,7 +71,8 @@ pub enum MbtType { Normalized, } -#[derive(PartialEq, Eq, Default, Debug, Clone)] +#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] +#[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(ValueEnum))] pub enum IntegrityCheckType { #[default] @@ -94,11 +104,13 @@ impl Mbtiles { } pub async fn open(&self) -> MbtResult { + debug!("Opening w/ defaults {}", self.filepath()); let opt = SqliteConnectOptions::new().filename(self.filepath()); Self::open_int(&opt).await } pub async fn open_or_new(&self) -> MbtResult { + debug!("Opening or creating {}", self.filepath()); let opt = SqliteConnectOptions::new() .filename(self.filepath()) .create_if_missing(true); @@ -106,6 +118,7 @@ impl Mbtiles { } pub async fn open_readonly(&self) -> MbtResult { + debug!("Opening as readonly {}", self.filepath()); let opt = SqliteConnectOptions::new() .filename(self.filepath()) .read_only(true); @@ -144,6 +157,7 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { + debug!("Attaching {} as {name}", self.filepath()); query(&format!("ATTACH DATABASE ? AS {name}")) .bind(self.filepath()) .execute(conn) @@ -171,7 +185,7 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { - self.get_metadata_value(&mut *conn, "agg_tiles_hash").await + self.get_metadata_value(&mut *conn, AGG_TILES_HASH).await } pub async fn set_metadata_value( @@ -397,6 +411,7 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { + debug!("Detecting MBTiles type for {}", self.filepath()); let mbt_type = if is_normalized_tables_type(&mut *conn).await? { MbtType::Normalized } else if is_flat_with_hash_tables_type(&mut *conn).await? { @@ -534,7 +549,7 @@ impl Mbtiles { } else { info!("Creating new metadata value agg_tiles_hash = {hash} in {path}"); } - self.set_metadata_value(&mut *conn, "agg_tiles_hash", Some(hash)) + self.set_metadata_value(&mut *conn, AGG_TILES_HASH, Some(hash)) .await } } @@ -546,7 +561,7 @@ impl Mbtiles { // Note that hex() always returns upper-case HEX values let sql = match self.detect_type(&mut *conn).await? { MbtType::Flat => { - println!("Skipping per-tile hash validation because this is a flat MBTiles file"); + info!("Skipping per-tile hash validation because this is a flat MBTiles file"); return Ok(()); } MbtType::FlatWithHash => { @@ -593,6 +608,7 @@ async fn calc_agg_tiles_hash(conn: &mut T) -> MbtResult where for<'e> &'e mut T: SqliteExecutor<'e>, { + debug!("Calculating agg_tiles_hash"); let query = query( // The md5_concat func will return NULL if there are no rows in the tiles table. // For our use case, we will treat it as an empty string, and hash that. @@ -626,6 +642,17 @@ pub async fn attach_hash_fn(conn: &mut SqliteConnection) -> MbtResult<()> { Ok(()) } +pub async fn detach_db(conn: &mut T, name: &str) -> MbtResult<()> +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + debug!("Detaching {name}"); + query(&format!("DETACH DATABASE {name}")) + .execute(conn) + .await?; + Ok(()) +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -638,9 +665,7 @@ mod tests { async fn open(filepath: &str) -> MbtResult<(SqliteConnection, Mbtiles)> { let mbt = Mbtiles::new(filepath)?; - let mut conn = SqliteConnection::connect(mbt.filepath()).await?; - attach_hash_fn(&mut conn).await?; - Ok((conn, mbt)) + mbt.open().await.map(|conn| (conn, mbt)) } #[actix_rt::test] diff --git a/martin-mbtiles/src/queries.rs b/martin-mbtiles/src/queries.rs index 2986dab69..8fc539ad3 100644 --- a/martin-mbtiles/src/queries.rs +++ b/martin-mbtiles/src/queries.rs @@ -1,3 +1,4 @@ +use log::debug; use sqlx::{query, Executor as _, SqliteExecutor}; use crate::errors::MbtResult; @@ -125,6 +126,7 @@ pub async fn create_metadata_table(conn: &mut T) -> MbtResult<()> where for<'e> &'e mut T: SqliteExecutor<'e>, { + debug!("Creating metadata table if it doesn't already exist"); conn.execute( "CREATE TABLE IF NOT EXISTS metadata ( name text NOT NULL PRIMARY KEY, @@ -141,6 +143,7 @@ where { create_metadata_table(&mut *conn).await?; + debug!("Creating if needed flat table: tiles(z,x,y,data)"); conn.execute( "CREATE TABLE IF NOT EXISTS tiles ( zoom_level integer NOT NULL, @@ -160,6 +163,7 @@ where { create_metadata_table(&mut *conn).await?; + debug!("Creating if needed flat-with-hash table: tiles_with_hash(z,x,y,data,hash)"); conn.execute( "CREATE TABLE IF NOT EXISTS tiles_with_hash ( zoom_level integer NOT NULL, @@ -171,6 +175,7 @@ where ) .await?; + debug!("Creating if needed tiles view for flat-with-hash"); conn.execute( "CREATE VIEW IF NOT EXISTS tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;", @@ -186,6 +191,7 @@ where { create_metadata_table(&mut *conn).await?; + debug!("Creating if needed normalized table: map(z,x,y,id)"); conn.execute( "CREATE TABLE IF NOT EXISTS map ( zoom_level integer NOT NULL, @@ -196,6 +202,7 @@ where ) .await?; + debug!("Creating if needed normalized table: images(id,data)"); conn.execute( "CREATE TABLE IF NOT EXISTS images ( tile_data blob, @@ -203,6 +210,7 @@ where ) .await?; + debug!("Creating if needed tiles view for flat-with-hash"); conn.execute( "CREATE VIEW IF NOT EXISTS tiles AS SELECT map.zoom_level AS zoom_level, @@ -221,6 +229,7 @@ pub async fn create_tiles_with_hash_view(conn: &mut T) -> MbtResult<()> where for<'e> &'e mut T: SqliteExecutor<'e>, { + debug!("Creating if needed tiles_with_hash view for normalized map+images structure"); conn.execute( "CREATE VIEW IF NOT EXISTS tiles_with_hash AS SELECT diff --git a/martin-mbtiles/tests/mbtiles.rs b/martin-mbtiles/tests/mbtiles.rs new file mode 100644 index 000000000..def670d45 --- /dev/null +++ b/martin-mbtiles/tests/mbtiles.rs @@ -0,0 +1,281 @@ +use std::path::PathBuf; +use std::str::from_utf8; + +use ctor::ctor; +use insta::assert_toml_snapshot; +use martin_mbtiles::MbtType::{Flat, FlatWithHash, Normalized}; +use martin_mbtiles::{apply_diff, create_flat_tables, MbtResult, Mbtiles, MbtilesCopier}; +use serde::Serialize; +use sqlx::{query, query_as, Executor as _, Row, SqliteConnection}; + +const INSERT_TILES_V1: &str = " + INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES + (1, 0, 0, cast('same' as blob)), + (1, 0, 1, cast('edit-v1' as blob)), + (1, 1, 1, cast('remove' as blob))"; + +const INSERT_TILES_V2: &str = " + INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES + (1, 0, 0, cast('same' as blob)), + (1, 0, 1, cast('edit-v2' as blob)), + (1, 1, 0, cast('new' as blob))"; + +const INSERT_METADATA_V1: &str = " + INSERT INTO metadata (name, value) VALUES + ('same', 'value - same'), + ('edit', 'value - v1'), + ('remove', 'value - remove')"; + +const INSERT_METADATA_V2: &str = " + INSERT INTO metadata (name, value) VALUES + ('same', 'value - same'), + ('edit', 'value - v2'), + ('new', 'value - new')"; + +#[ctor] +fn init() { + let _ = env_logger::builder().is_test(true).try_init(); +} + +fn path(mbt: &Mbtiles) -> PathBuf { + PathBuf::from(mbt.filepath()) +} + +fn copier(src: &Mbtiles, dst: &Mbtiles) -> MbtilesCopier { + MbtilesCopier::new(path(src), path(dst)) +} + +async fn open(file: &str) -> MbtResult<(Mbtiles, SqliteConnection)> { + let mbtiles = Mbtiles::new(file)?; + let conn = mbtiles.open().await?; + Ok((mbtiles, conn)) +} + +macro_rules! assert_snapshot { + ($prefix:expr, $name:expr, $actual:expr) => { + let mut settings = insta::Settings::clone_current(); + let name = if $name != "" { + format!("{}-{}", $prefix, $name) + } else { + $prefix.to_string() + }; + settings.set_snapshot_suffix(name); + let result = $actual; + settings.bind(|| assert_toml_snapshot!(result)); + }; +} + +macro_rules! open { + ($function:tt, $name:expr) => {{ + let func = stringify!($function); + let file = format!("file:{func}_{}?mode=memory&cache=shared", $name); + open(&file).await? + }}; +} + +macro_rules! new_source { + ($function:tt, $dst:expr, $sql_meta:expr, $sql_data:expr) => {{ + let func = stringify!($function); + let name = stringify!($dst); + let file = format!("file:{func}_{name}?mode=memory&cache=shared"); + let (dst, mut cn_dst) = open(&file).await?; + create_flat_tables(&mut cn_dst).await?; + cn_dst.execute($sql_data).await?; + cn_dst.execute($sql_meta).await?; + assert_snapshot!(name, "", dump(&mut cn_dst).await?); + (dst, cn_dst) + }}; +} + +/// Copy SQLite from $src to $dst, and add agg_tiles_hash metadata value +/// Returns the destination Mbtiles, the SQLite connection, and the dump of the SQLite +macro_rules! copy_hash { + ($function:tt, $src:expr, $dst:tt, $dst_type:expr) => {{ + copy_to!($function, $src, $dst, $dst_type, false) + }}; +} + +/// Copy SQLite from $src to $dst, no changes by default, or add agg_tiles_hash metadata if last arg is false +/// The result is dumped to a snapshot +/// Returns the destination Mbtiles, the SQLite connection, and the dump of the SQLite +macro_rules! copy_to { + ($function:tt, $src:expr, $dst:tt, $dst_type:expr) => {{ + copy_to!($function, $src, $dst, $dst_type, true) + }}; + ($function:tt, $src:expr, $dst:tt, $dst_type:expr, $skip_agg:expr) => {{ + let func = stringify!($function); + let name = stringify!($dst); + let file = format!("file:{func}_{name}?mode=memory&cache=shared"); + let (dst, cn_dst) = open(&file).await?; + let mut opt = copier(&$src, &dst); + opt.skip_agg_tiles_hash = $skip_agg; + opt.dst_type = Some($dst_type); + let dmp = dump(&mut opt.run().await?).await?; + assert_snapshot!(name, "", &dmp); + (dst, cn_dst, dmp) + }}; +} + +#[actix_rt::test] +async fn copy_and_convert() -> MbtResult<()> { + let mem = Mbtiles::new(":memory:")?; + + let (orig, _cn_orig) = new_source!(test_copy, orig, INSERT_METADATA_V1, INSERT_TILES_V1); + let (flat, _cn_flat, _) = copy_to!(test_copy, orig, flat, Flat); + let (hash, _cn_hash, _) = copy_to!(test_copy, orig, hash, FlatWithHash); + let (norm, _cn_norm, _) = copy_to!(test_copy, orig, norm, Normalized); + + for (frm, src) in &[("flat", &flat), ("hash", &hash), ("norm", &norm)] { + // Same content, but also will include agg_tiles_hash metadata value + let opt = copier(src, &mem); + assert_snapshot!(frm, "cp", dump(&mut opt.run().await?).await?); + + // Identical content to the source + let mut opt = copier(src, &mem); + opt.skip_agg_tiles_hash = true; + assert_snapshot!(frm, "cp-skip-agg", dump(&mut opt.run().await?).await?); + + // Copy to a flat-with-hash schema + let mut opt = copier(src, &mem); + opt.dst_type = Some(FlatWithHash); + assert_snapshot!(frm, "to-hash", dump(&mut opt.run().await?).await?); + + // Copy to a flat schema + let mut opt = copier(src, &mem); + opt.dst_type = Some(Flat); + assert_snapshot!(frm, "to-flat", dump(&mut opt.run().await?).await?); + + // Copy to a normalized schema + let mut opt = copier(src, &mem); + opt.dst_type = Some(Normalized); + assert_snapshot!(frm, "to-norm", dump(&mut opt.run().await?).await?); + } + + Ok(()) +} + +/// Create v1 and v2 in-memory sqlite files, and copy them to flat, hash, and norm variants +/// Keep connection open to make sure they can be opened as regular files by the main code. +/// For different variants, create a delta v2-v1, and re-apply the diff to v1 to get v2a - which should be identical to v2. +#[actix_rt::test] +async fn diff_and_apply() -> MbtResult<()> { + let (orig_v1, _cn_orig_v1) = + new_source!(test_diff, orig_v1, INSERT_METADATA_V1, INSERT_TILES_V1); + let (flat_v1, _cn_flat_v1, _) = copy_hash!(test_diff, orig_v1, flat_v1, Flat); + let (hash_v1, _cn_hash_v1, _) = copy_hash!(test_diff, orig_v1, hash_v1, FlatWithHash); + let (norm_v1, _cn_norm_v1, _) = copy_hash!(test_diff, orig_v1, norm_v1, Normalized); + + let (orig_v2, _cn_orig_v2) = + new_source!(test_diff, orig_v2, INSERT_METADATA_V2, INSERT_TILES_V2); + let (flat_v2, _cn_flat_v2, dmp_flat_v2) = copy_hash!(test_diff, orig_v2, flat_v2, Flat); + let (hash_v2, _cn_hash_v2, dmp_hash_v2) = copy_hash!(test_diff, orig_v2, hash_v2, FlatWithHash); + let (norm_v2, _cn_norm_v2, dmp_norm_v2) = copy_hash!(test_diff, orig_v2, norm_v2, Normalized); + + for (frm, v1, v2, dump_v2) in &[ + ("flat", &flat_v1, &flat_v2, dmp_flat_v2), + ("hash", &hash_v1, &hash_v2, dmp_hash_v2), + ("norm", &norm_v1, &norm_v2, dmp_norm_v2), + ] { + let (dff, _cn_dff) = open!(test_diff, format!("{frm}-dff")); + let (v2a, mut cn_v2a) = open!(test_diff, format!("{frm}-v2a")); + + // Diff v1 with v2, and copy to diff anything that's different (i.e. mathematically: v2-v1) + let mut diff_with = copier(v1, &dff); + diff_with.diff_with_file = Some(path(v2)); + assert_snapshot!(frm, "diff_v2-v1", dump(&mut diff_with.run().await?).await?); + + // Copy v1 -> v2a, and apply dff to v2a + copier(v1, &v2a).run().await?; + apply_diff(path(&v2a), path(&dff)).await?; + + let dump_v2a = dump(&mut cn_v2a).await?; + assert_snapshot!(frm, "applied-diff", &dump_v2a); + + // FIXME!!! + if frm != &"norm" { + pretty_assertions::assert_eq!( + &dump_v2a, + dump_v2, + "v2a should be identical to v2 (type = {frm})" + ); + } + } + + Ok(()) +} + +#[derive(Debug, sqlx::FromRow, Serialize, PartialEq)] +struct SqliteEntry { + pub r#type: Option, + pub tbl_name: Option, + pub sql: Option, + #[sqlx(skip)] + pub values: Option>, +} + +async fn dump(conn: &mut SqliteConnection) -> MbtResult> { + let mut result = Vec::new(); + + let schema: Vec = query_as( + "SELECT type, tbl_name, sql + FROM sqlite_schema + ORDER BY type != 'table', type, tbl_name", + ) + .fetch_all(&mut *conn) + .await?; + + for mut entry in schema { + let tbl = match (&entry.r#type, &entry.tbl_name) { + (Some(typ), Some(tbl)) if typ == "table" => tbl, + _ => { + result.push(entry); + continue; + } + }; + + let sql = format!("PRAGMA table_info({tbl})"); + let columns: Vec<_> = query(&sql) + .fetch_all(&mut *conn) + .await? + .into_iter() + .map(|row| { + let cid: i32 = row.get(0); + let typ: String = row.get(2); + (cid as usize, typ) + }) + .collect(); + + let sql = format!("SELECT * FROM {tbl}"); + let rows = query(&sql).fetch_all(&mut *conn).await?; + let mut values = rows + .iter() + .map(|row| { + let val = columns + .iter() + .map(|(idx, typ)| { + (match typ.as_str() { + "INTEGER" => row.get::, _>(idx).map(|v| v.to_string()), + "REAL" => row.get::, _>(idx).map(|v| v.to_string()), + "TEXT" => row + .get::, _>(idx) + .map(|v| format!(r#""{v}""#)), + "BLOB" => row + .get::>, _>(idx) + .map(|v| format!(r#""{}""#, from_utf8(&v).unwrap())), + _ => panic!("Unknown column type: {typ}"), + }) + .unwrap_or("NULL".to_string()) + }) + .collect::>() + .join(", "); + format!("( {val} )") + }) + .collect::>(); + + values.sort(); + entry.values = Some(values); + result.push(entry); + } + + Ok(result) +} diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-cp-skip-agg.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-cp-skip-agg.snap new file mode 100644 index 000000000..38b7767bd --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-cp-skip-agg.snap @@ -0,0 +1,40 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-cp.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-cp.snap new file mode 100644 index 000000000..897d9bd64 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-cp.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-flat.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-flat.snap new file mode 100644 index 000000000..897d9bd64 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-flat.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-hash.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-hash.snap new file mode 100644 index 000000000..cfaeac66e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-hash.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-norm.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-norm.snap new file mode 100644 index 000000000..7d3ed414e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat-to-norm.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat.snap new file mode 100644 index 000000000..38b7767bd --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@flat.snap @@ -0,0 +1,40 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-cp-skip-agg.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-cp-skip-agg.snap new file mode 100644 index 000000000..92ddb098d --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-cp-skip-agg.snap @@ -0,0 +1,48 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-cp.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-cp.snap new file mode 100644 index 000000000..cfaeac66e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-cp.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-flat.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-flat.snap new file mode 100644 index 000000000..897d9bd64 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-flat.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-hash.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-hash.snap new file mode 100644 index 000000000..cfaeac66e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-hash.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-norm.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-norm.snap new file mode 100644 index 000000000..7d3ed414e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash-to-norm.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash.snap new file mode 100644 index 000000000..92ddb098d --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@hash.snap @@ -0,0 +1,48 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-cp-skip-agg.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-cp-skip-agg.snap new file mode 100644 index 000000000..3437bfb67 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-cp-skip-agg.snap @@ -0,0 +1,83 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-cp.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-cp.snap new file mode 100644 index 000000000..7d3ed414e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-cp.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-flat.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-flat.snap new file mode 100644 index 000000000..897d9bd64 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-flat.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-hash.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-hash.snap new file mode 100644 index 000000000..cfaeac66e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-hash.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-norm.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-norm.snap new file mode 100644 index 000000000..7d3ed414e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm-to-norm.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm.snap new file mode 100644 index 000000000..3437bfb67 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@norm.snap @@ -0,0 +1,83 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@orig.snap b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@orig.snap new file mode 100644 index 000000000..38b7767bd --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__copy_and_convert@orig.snap @@ -0,0 +1,40 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat-applied-diff.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat-applied-diff.snap new file mode 100644 index 000000000..08a28423a --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat-applied-diff.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v2" )', + '( 1, 1, 0, "new" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat-diff_v2-v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat-diff_v2-v1.snap new file mode 100644 index 000000000..905c82dac --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat-diff_v2-v1.snap @@ -0,0 +1,42 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "B69186152AEB4709BB5E51F26122506D" )', + '( "agg_tiles_hash_after_apply", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "remove", NULL )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 1, "edit-v2" )', + '( 1, 1, 0, "new" )', + '( 1, 1, 1, NULL )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat_v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat_v1.snap new file mode 100644 index 000000000..897d9bd64 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat_v1.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat_v2.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat_v2.snap new file mode 100644 index 000000000..08a28423a --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@flat_v2.snap @@ -0,0 +1,41 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v2" )', + '( 1, 1, 0, "new" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash-applied-diff.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash-applied-diff.snap new file mode 100644 index 000000000..dc624f498 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash-applied-diff.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v2", "FF76830FF90D79BB335884F256031731" )', + '( 1, 1, 0, "new", "22AF645D1859CB5CA6DA0C484F1F37EA" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash-diff_v2-v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash-diff_v2-v1.snap new file mode 100644 index 000000000..7ce4fbd4b --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash-diff_v2-v1.snap @@ -0,0 +1,50 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "B69186152AEB4709BB5E51F26122506D" )', + '( "agg_tiles_hash_after_apply", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "remove", NULL )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 1, "edit-v2", "FF76830FF90D79BB335884F256031731" )', + '( 1, 1, 0, "new", "22AF645D1859CB5CA6DA0C484F1F37EA" )', + '( 1, 1, 1, NULL, NULL )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash_v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash_v1.snap new file mode 100644 index 000000000..cfaeac66e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash_v1.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash_v2.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash_v2.snap new file mode 100644 index 000000000..dc624f498 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@hash_v2.snap @@ -0,0 +1,49 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same", "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "edit-v2", "FF76830FF90D79BB335884F256031731" )', + '( 1, 1, 0, "new", "22AF645D1859CB5CA6DA0C484F1F37EA" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm-applied-diff.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm-applied-diff.snap new file mode 100644 index 000000000..4c0d3c222 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm-applied-diff.snap @@ -0,0 +1,86 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "edit-v2", "FF76830FF90D79BB335884F256031731" )', + '( "new", "22AF645D1859CB5CA6DA0C484F1F37EA" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "FF76830FF90D79BB335884F256031731" )', + '( 1, 1, 0, "22AF645D1859CB5CA6DA0C484F1F37EA" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm-diff_v2-v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm-diff_v2-v1.snap new file mode 100644 index 000000000..c2b60362e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm-diff_v2-v1.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v2", "FF76830FF90D79BB335884F256031731" )', + '( "new", "22AF645D1859CB5CA6DA0C484F1F37EA" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 1, "FF76830FF90D79BB335884F256031731" )', + '( 1, 1, 0, "22AF645D1859CB5CA6DA0C484F1F37EA" )', + '( 1, 1, 1, NULL )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "5E37A28AFE35EA0BFF6703CB3ECE8B2F" )', + '( "agg_tiles_hash_after_apply", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "remove", NULL )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm_v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm_v1.snap new file mode 100644 index 000000000..7d3ed414e --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm_v1.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v1", "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( "remove", "0F6969D7052DA9261E31DDB6E88C136E" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 1, 1, 1, "0F6969D7052DA9261E31DDB6E88C136E" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "810E134831F46B3A60D425EF11B5FBE9" )', + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm_v2.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm_v2.snap new file mode 100644 index 000000000..5a067a3ac --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@norm_v2.snap @@ -0,0 +1,84 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_data blob, + tile_id text NOT NULL PRIMARY KEY)''' +values = [ + '( "edit-v2", "FF76830FF90D79BB335884F256031731" )', + '( "new", "22AF645D1859CB5CA6DA0C484F1F37EA" )', + '( "same", "51037A4A37730F52C8732586D3AAA316" )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 1, 0, 1, "FF76830FF90D79BB335884F256031731" )', + '( 1, 1, 0, "22AF645D1859CB5CA6DA0C484F1F37EA" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "4CB1879C4A84910F25B9272B6E5B6E7C" )', + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@orig_v1.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@orig_v1.snap new file mode 100644 index 000000000..38b7767bd --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@orig_v1.snap @@ -0,0 +1,40 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v1" )', + '( "remove", "value - remove" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v1" )', + '( 1, 1, 1, "remove" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@orig_v2.snap b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@orig_v2.snap new file mode 100644 index 000000000..80434e261 --- /dev/null +++ b/martin-mbtiles/tests/snapshots/mbtiles__diff_and_apply@orig_v2.snap @@ -0,0 +1,40 @@ +--- +source: martin-mbtiles/tests/mbtiles.rs +expression: result +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "edit", "value - v2" )', + '( "new", "value - new" )', + '( "same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 1, 0, 0, "same" )', + '( 1, 0, 1, "edit-v2" )', + '( 1, 1, 0, "new" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/martin/src/pg/table_source.rs b/martin/src/pg/table_source.rs index 9fe61a913..d26adad85 100644 --- a/martin/src/pg/table_source.rs +++ b/martin/src/pg/table_source.rs @@ -261,8 +261,6 @@ pub fn calc_srid( Some(default_srid) } (0, 0, None) => { - // TODO: cleanup - // println!("{:#?}", std::backtrace::Backtrace::force_capture()); let info = "To use this table source, set default or specify this table SRID in the config file, or set the default SRID with --default-srid=..."; warn!("Table {table_id} has SRID=0, skipping. {info}"); None diff --git a/tests/expected/mbtiles/copy_diff.txt b/tests/expected/mbtiles/copy_diff.txt index 4fc7c848d..615cfdffc 100644 --- a/tests/expected/mbtiles/copy_diff.txt +++ b/tests/expected/mbtiles/copy_diff.txt @@ -1 +1,3 @@ +[INFO ] Copying ./tests/fixtures/mbtiles/world_cities.mbtiles (flat) to a new file tests/temp/world_cities_diff.mbtiles (flat) +[INFO ] Copying only the data not found in ./tests/fixtures/mbtiles/world_cities_modified.mbtiles (flat) [INFO ] Creating new metadata value agg_tiles_hash = C7E2E5A9BA04693994DB1F57D1DF5646 in tests/temp/world_cities_diff.mbtiles diff --git a/tests/expected/mbtiles/copy_diff2.txt b/tests/expected/mbtiles/copy_diff2.txt index cddb2637d..4d36c0614 100644 --- a/tests/expected/mbtiles/copy_diff2.txt +++ b/tests/expected/mbtiles/copy_diff2.txt @@ -1 +1,3 @@ +[INFO ] Copying ./tests/fixtures/mbtiles/world_cities_modified.mbtiles (flat) to a new file tests/temp/world_cities_diff_modified.mbtiles (flat) +[INFO ] Copying only the data not found in tests/temp/world_cities_copy.mbtiles (flat) [INFO ] Creating new metadata value agg_tiles_hash = D41D8CD98F00B204E9800998ECF8427E in tests/temp/world_cities_diff_modified.mbtiles diff --git a/tests/fixtures/mbtiles/world_cities.mbties b/tests/fixtures/mbtiles/world_cities.mbties new file mode 100644 index 000000000..e69de29bb diff --git a/tests/pg_server_test.rs b/tests/pg_server_test.rs index 1d65869c3..7f4037a15 100644 --- a/tests/pg_server_test.rs +++ b/tests/pg_server_test.rs @@ -877,7 +877,6 @@ postgres: let req = test_get("/function_zxy_query_test/0/0/0"); let response = call_service(&app, req).await; - println!("response.status = {:?}", response.status()); assert!(response.status().is_server_error()); let req = test_get("/function_zxy_query_test/0/0/0?token=martin"); @@ -990,7 +989,9 @@ tables: ) .await; - let OneOrMany::One(cfg) = cfg.postgres.unwrap() else { panic!() }; + let OneOrMany::One(cfg) = cfg.postgres.unwrap() else { + panic!() + }; for (name, _) in cfg.tables.unwrap_or_default() { let req = test_get(format!("/{name}/0/0/0").as_str()); let response = call_service(&app, req).await;