diff --git a/Cargo.toml b/Cargo.toml index 1bd28c932..8bbd2d30b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,3 +69,4 @@ similar.opt-level = 3 #[patch.crates-io] #sqlite-hashes = { path = "/home/nyurik/dev/rust/sqlite-hashes" } +#pmtiles = { path = "../pmtiles-rs" } diff --git a/docs/src/30-config-file.md b/docs/src/30-config-file.md index a4df42845..99817a061 100644 --- a/docs/src/30-config-file.md +++ b/docs/src/30-config-file.md @@ -158,11 +158,11 @@ pmtiles: paths: # scan this whole dir, matching all *.pmtiles files - /dir-path - # specific pmtiles file will be published as pmtiles2 source - - /path/to/pmtiles.pmtiles + # specific pmtiles file will be published as a pmt source (filename without extension) + - /path/to/pmt.pmtiles sources: # named source matching source name to a single file - pm-src1: /path/to/pmtiles1.pmtiles + pm-src1: /path/to/pmt.pmtiles # Publish MBTiles files mbtiles: diff --git a/justfile b/justfile index 873c8aa9b..c17c6de58 100644 --- a/justfile +++ b/justfile @@ -3,6 +3,7 @@ set shell := ["bash", "-c"] #export DATABASE_URL="postgres://postgres:postgres@localhost:5411/db" + export PGPORT := "5411" export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db" export CARGO_TERM_COLOR := "always" @@ -12,7 +13,7 @@ export CARGO_TERM_COLOR := "always" #export RUST_BACKTRACE := "1" @_default: - {{just_executable()}} --list --unsorted + {{ just_executable() }} --list --unsorted # Start Martin server run *ARGS: @@ -29,7 +30,7 @@ run-release *ARGS: start # Start Martin server and open a test page debug-page *ARGS: start open tests/debug.html # run will not exit, so open debug page first - {{just_executable()}} run {{ ARGS }} + {{ just_executable() }} run {{ ARGS }} # Run PSQL utility against the test database psql *ARGS: @@ -75,8 +76,9 @@ alias _stop-db := stop # Restart the test database restart: - {{just_executable()}} stop - {{just_executable()}} start + # sometimes Just optimizes targets, so here we force stop & start by using external just executable + {{ just_executable() }} stop + {{ just_executable() }} start # Stop the test database stop: @@ -113,9 +115,9 @@ test-ssl-cert: start-ssl-cert export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem" export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem" export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key" - {{just_executable()}} test-cargo --all-targets - {{just_executable()}} clean-test - {{just_executable()}} test-doc + {{ just_executable() }} test-cargo --all-targets + {{ just_executable() }} clean-test + {{ just_executable() }} test-doc tests/test.sh # Run all tests using the oldest supported version of the database @@ -134,12 +136,18 @@ test-int: clean-test install-sqlx #!/usr/bin/env bash set -euo pipefail tests/test.sh - if ! diff --brief --recursive --new-file tests/output tests/expected; then - echo "** Expected output does not match actual output" - echo "** If this is expected, run 'just bless' to update expected output" - exit 1 + if [ "{{ os() }}" != "linux" ]; then + echo "** Integration tests are only supported on Linux" + echo "** Skipping diffing with the expected output" else - echo "Expected output matches actual output" + echo "** Comparing actual output with expected output..." + if ! diff --brief --recursive --new-file tests/output tests/expected; then + echo "** Expected output does not match actual output" + echo "** If this is expected, run 'just bless' to update expected output" + exit 1 + else + echo "** Expected output matches actual output" + fi fi # Run integration tests and save its output as the new expected output @@ -180,8 +188,8 @@ coverage FORMAT='html': (cargo-install "grcov") rustup component add llvm-tools-preview ;\ fi - {{just_executable()}} clean - {{just_executable()}} start + {{ just_executable() }} clean + {{ just_executable() }} start PROF_DIR=target/prof mkdir -p "$PROF_DIR" @@ -252,11 +260,15 @@ clippy: # These steps automatically run before git push via a git hook [private] -git-pre-push: stop start +git-pre-push: env-info restart lint test + +# Get environment info +[private] +env-info: + @echo "OS is {{ os() }}, arch is {{ arch() }}" + {{ just_executable() }} --version rustc --version cargo --version - {{just_executable()}} lint - {{just_executable()}} test # Update sqlite database schema. prepare-sqlite: install-sqlx diff --git a/martin/src/config.rs b/martin/src/config.rs index 625485e01..e165b8bca 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -90,8 +90,8 @@ impl Config { } async fn resolve_tile_sources(&mut self, idr: IdResolver) -> Result { - let create_pmt_src = &mut PmtSource::new_box; - let create_mbt_src = &mut MbtSource::new_box; + let new_pmt_src = &mut PmtSource::new_box; + let new_mbt_src = &mut MbtSource::new_box; let mut sources: Vec>>>> = Vec::new(); for s in self.postgres.iter_mut() { @@ -99,12 +99,12 @@ impl Config { } if !self.pmtiles.is_empty() { - let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", create_pmt_src); + let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", new_pmt_src); sources.push(Box::pin(val)); } if !self.mbtiles.is_empty() { - let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", create_mbt_src); + let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", new_mbt_src); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 62d8ab30e..50a57e458 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -140,11 +140,24 @@ pub enum FileConfigSrc { } impl FileConfigSrc { - pub fn abs_path(&self) -> Result { - let path = match self { + #[must_use] + pub fn into_path(self) -> PathBuf { + match self { + Self::Path(p) => p, + Self::Obj(o) => o.path, + } + } + + #[must_use] + pub fn get_path(&self) -> &PathBuf { + match self { Self::Path(p) => p, Self::Obj(o) => &o.path, - }; + } + } + + pub fn abs_path(&self) -> Result { + let path = self.get_path(); path.canonicalize().map_err(|e| IoError(e, path.clone())) } } @@ -158,12 +171,12 @@ pub async fn resolve_files( config: &mut FileConfigEnum, idr: IdResolver, extension: &str, - create_source: &mut impl FnMut(String, PathBuf) -> Fut, + new_source: &mut impl FnMut(String, PathBuf) -> Fut, ) -> Result where Fut: Future, FileError>>, { - resolve_int(config, idr, extension, create_source) + resolve_int(config, idr, extension, new_source) .map_err(crate::Error::from) .await } @@ -172,7 +185,7 @@ async fn resolve_int( config: &mut FileConfigEnum, idr: IdResolver, extension: &str, - create_source: &mut impl FnMut(String, PathBuf) -> Fut, + new_source: &mut impl FnMut(String, PathBuf) -> Fut, ) -> Result where Fut: Future, FileError>>, @@ -204,7 +217,7 @@ where FileConfigSrc::Obj(pmt) => pmt.path, FileConfigSrc::Path(path) => path, }; - results.push(create_source(id, path).await?); + results.push(new_source(id, path).await?); } } @@ -248,7 +261,7 @@ where FileConfigSrc::Obj(pmt) => pmt.path, FileConfigSrc::Path(path) => path, }; - results.push(create_source(id, path).await?); + results.push(new_source(id, path).await?); } } @@ -288,7 +301,7 @@ mod tests { paths, vec![ PathBuf::from("/dir-path"), - PathBuf::from("/path/to/file2.ext") + PathBuf::from("/path/to/file2.ext"), ] ); assert_eq!( diff --git a/tests/config.yaml b/tests/config.yaml index cecc48aa7..06ed8f98b 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -163,6 +163,7 @@ postgres: pmtiles: sources: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + pmt2: tests/fixtures/pmtiles2/webp2.pmtiles sprites: paths: tests/fixtures/sprites/src1 diff --git a/tests/expected/auto/webp2_1_0_0.png b/tests/expected/auto/webp2_1_0_0.png new file mode 100644 index 000000000..6025b4536 Binary files /dev/null and b/tests/expected/auto/webp2_1_0_0.png differ diff --git a/tests/expected/auto/webp2_1_0_0.png.txt b/tests/expected/auto/webp2_1_0_0.png.txt new file mode 100644 index 000000000..706a01c7b --- /dev/null +++ b/tests/expected/auto/webp2_1_0_0.png.txt @@ -0,0 +1 @@ +tests/output/auto/webp2_1_0_0.png: RIFF (little-endian) data, Web/P image, VP8 encoding, 512x512, Scaling: [none]x[none], YUV color, decoders should clamp diff --git a/tests/expected/configured/catalog_cfg.json b/tests/expected/configured/catalog_cfg.json index 810687e76..affe0ae0e 100644 --- a/tests/expected/configured/catalog_cfg.json +++ b/tests/expected/configured/catalog_cfg.json @@ -23,6 +23,10 @@ "pmt": { "content_type": "image/png" }, + "pmt2": { + "content_type": "image/webp", + "name": "ne2sr" + }, "points1": { "content_type": "application/x-protobuf", "description": "public.points1.geom" diff --git a/tests/expected/configured/pmt2_0_0_0.png b/tests/expected/configured/pmt2_0_0_0.png new file mode 100644 index 000000000..543173081 Binary files /dev/null and b/tests/expected/configured/pmt2_0_0_0.png differ diff --git a/tests/expected/configured/pmt2_0_0_0.png.txt b/tests/expected/configured/pmt2_0_0_0.png.txt new file mode 100644 index 000000000..1587845ee --- /dev/null +++ b/tests/expected/configured/pmt2_0_0_0.png.txt @@ -0,0 +1 @@ +tests/output/configured/pmt2_0_0_0.png: RIFF (little-endian) data, Web/P image diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index 435d52f41..c2317e9ce 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -191,14 +191,16 @@ pmtiles: paths: - tests/fixtures/mbtiles - tests/fixtures/pmtiles + - tests/fixtures/pmtiles2 sources: png: tests/fixtures/pmtiles/png.pmtiles stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles - webp2: tests/fixtures/pmtiles/webp2.pmtiles + webp2: tests/fixtures/pmtiles2/webp2.pmtiles mbtiles: paths: - tests/fixtures/mbtiles - tests/fixtures/pmtiles + - tests/fixtures/pmtiles2 sources: geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles diff --git a/tests/expected/given_config.yaml b/tests/expected/given_config.yaml index 64291515e..3de20ebcb 100644 --- a/tests/expected/given_config.yaml +++ b/tests/expected/given_config.yaml @@ -160,6 +160,7 @@ postgres: pmtiles: sources: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles + pmt2: tests/fixtures/pmtiles2/webp2.pmtiles sprites: paths: tests/fixtures/sprites/src1 sources: diff --git a/tests/fixtures/pmtiles/webp2.pmtiles b/tests/fixtures/pmtiles2/webp2.pmtiles similarity index 100% rename from tests/fixtures/pmtiles/webp2.pmtiles rename to tests/fixtures/pmtiles2/webp2.pmtiles diff --git a/tests/test.sh b/tests/test.sh index 9e76449d8..1423385b2 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -175,7 +175,7 @@ echo "Test auto configured Martin" TEST_OUT_DIR="$(dirname "$0")/output/auto" mkdir -p "$TEST_OUT_DIR" -ARG=(--default-srid 900913 --auto-bounds calc --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts) +ARG=(--default-srid 900913 --auto-bounds calc --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2 --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts) set -x $MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${LOG_DIR}/test_log_1.txt" & MARTIN_PROC_ID=`jobs -p | tail -n 1` @@ -237,8 +237,9 @@ test_jsn points3857_srid points3857 test_pbf points3857_srid_0_0_0 points3857/0/0/0 >&2 echo "***** Test server response for PMTiles source *****" -test_jsn pmt stamen_toner__raster_CC-BY-ODbL_z3 -test_png pmt_3_4_2 stamen_toner__raster_CC-BY-ODbL_z3/3/4/2 +test_jsn pmt stamen_toner__raster_CC-BY-ODbL_z3 +test_png pmt_3_4_2 stamen_toner__raster_CC-BY-ODbL_z3/3/4/2 +test_png webp2_1_0_0 webp2/1/0/0 # HTTP pmtiles >&2 echo "***** Test server response for MbTiles source *****" test_jsn mb_jpg geography-class-jpg @@ -276,6 +277,7 @@ test_pbf cmp_0_0_0 points1,points2/0/0/0 test_pbf fnc_0_0_0 function_zxy_query/0/0/0 test_pbf fnc2_0_0_0 function_zxy_query_test/0/0/0?token=martin test_png pmt_0_0_0 pmt/0/0/0 +test_png pmt2_0_0_0 pmt2/0/0/0 # HTTP pmtiles test_jsn spr_src1 sprite/src1.json test_png spr_src1 sprite/src1.png