Minor cleanup and just reorg

Only run git push diff on linux to make development on non-linux easier, and a few preparations for pmtiles over http serving

Cargo.toml

@@ -69,3 +69,4 @@ similar.opt-level = 3
 
 #[patch.crates-io]
 #sqlite-hashes = { path = "/home/nyurik/dev/rust/sqlite-hashes" }
+#pmtiles = { path = "../pmtiles-rs" }

docs/src/30-config-file.md

@@ -158,11 +158,11 @@ pmtiles:
   paths:
     # scan this whole dir, matching all *.pmtiles files
     - /dir-path
-    # specific pmtiles file will be published as pmtiles2 source
-    - /path/to/pmtiles.pmtiles
+    # specific pmtiles file will be published as a pmt source (filename without extension)
+    - /path/to/pmt.pmtiles
   sources:
     # named source matching source name to a single file
-    pm-src1: /path/to/pmtiles1.pmtiles
+    pm-src1: /path/to/pmt.pmtiles
 
 # Publish MBTiles files
 mbtiles:

justfile

@@ -3,6 +3,7 @@
 set shell := ["bash", "-c"]
 
 #export DATABASE_URL="postgres://postgres:postgres@localhost:5411/db"
+
 export PGPORT := "5411"
 export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db"
 export CARGO_TERM_COLOR := "always"
@@ -12,7 +13,7 @@ export CARGO_TERM_COLOR := "always"
 #export RUST_BACKTRACE := "1"
 
 @_default:
-    {{just_executable()}} --list --unsorted
+    {{ just_executable() }} --list --unsorted
 
 # Start Martin server
 run *ARGS:
@@ -29,7 +30,7 @@ run-release *ARGS: start
 # Start Martin server and open a test page
 debug-page *ARGS: start
     open tests/debug.html  # run will not exit, so open debug page first
-    {{just_executable()}} run {{ ARGS }}
+    {{ just_executable() }} run {{ ARGS }}
 
 # Run PSQL utility against the test database
 psql *ARGS:
@@ -75,8 +76,9 @@ alias _stop-db := stop
 
 # Restart the test database
 restart:
-    {{just_executable()}} stop
-    {{just_executable()}} start
+    # sometimes Just optimizes targets, so here we force stop & start by using external just executable
+    {{ just_executable() }} stop
+    {{ just_executable() }} start
 
 # Stop the test database
 stop:
@@ -113,9 +115,9 @@ test-ssl-cert: start-ssl-cert
     export PGSSLROOTCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
     export PGSSLCERT="$KEY_DIR/ssl-cert-snakeoil.pem"
     export PGSSLKEY="$KEY_DIR/ssl-cert-snakeoil.key"
-    {{just_executable()}} test-cargo --all-targets
-    {{just_executable()}} clean-test
-    {{just_executable()}} test-doc
+    {{ just_executable() }} test-cargo --all-targets
+    {{ just_executable() }} clean-test
+    {{ just_executable() }} test-doc
     tests/test.sh
 
 # Run all tests using the oldest supported version of the database
@@ -134,12 +136,18 @@ test-int: clean-test install-sqlx
     #!/usr/bin/env bash
     set -euo pipefail
     tests/test.sh
-    if ! diff --brief --recursive --new-file tests/output tests/expected; then
-        echo "** Expected output does not match actual output"
-        echo "** If this is expected, run 'just bless' to update expected output"
-        exit 1
+    if [ "{{ os() }}" != "linux" ]; then
+        echo "** Integration tests are only supported on Linux"
+        echo "** Skipping diffing with the expected output"
     else
-        echo "Expected output matches actual output"
+        echo "** Comparing actual output with expected output..."
+        if ! diff --brief --recursive --new-file tests/output tests/expected; then
+            echo "** Expected output does not match actual output"
+            echo "** If this is expected, run 'just bless' to update expected output"
+            exit 1
+        else
+            echo "** Expected output matches actual output"
+        fi
     fi
 
 # Run integration tests and save its output as the new expected output
@@ -180,8 +188,8 @@ coverage FORMAT='html': (cargo-install "grcov")
         rustup component add llvm-tools-preview ;\
     fi
 
-    {{just_executable()}} clean
-    {{just_executable()}} start
+    {{ just_executable() }} clean
+    {{ just_executable() }} start
 
     PROF_DIR=target/prof
     mkdir -p "$PROF_DIR"
@@ -252,11 +260,15 @@ clippy:
 
 # These steps automatically run before git push via a git hook
 [private]
-git-pre-push: stop start
+git-pre-push: env-info restart lint test
+
+# Get environment info
+[private]
+env-info:
+    @echo "OS is {{ os() }}, arch is {{ arch() }}"
+    {{ just_executable() }} --version
     rustc --version
     cargo --version
-    {{just_executable()}} lint
-    {{just_executable()}} test
 
 # Update sqlite database schema.
 prepare-sqlite: install-sqlx

martin/src/config.rs

@@ -90,21 +90,21 @@ impl Config {
     }
 
     async fn resolve_tile_sources(&mut self, idr: IdResolver) -> Result<TileSources> {
-        let create_pmt_src = &mut PmtSource::new_box;
-        let create_mbt_src = &mut MbtSource::new_box;
+        let new_pmt_src = &mut PmtSource::new_box;
+        let new_mbt_src = &mut MbtSource::new_box;
         let mut sources: Vec<Pin<Box<dyn Future<Output = Result<TileInfoSources>>>>> = Vec::new();
 
         for s in self.postgres.iter_mut() {
             sources.push(Box::pin(s.resolve(idr.clone())));
         }
 
         if !self.pmtiles.is_empty() {
-            let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", create_pmt_src);
+            let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", new_pmt_src);
             sources.push(Box::pin(val));
         }
 
         if !self.mbtiles.is_empty() {
-            let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", create_mbt_src);
+            let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", new_mbt_src);
             sources.push(Box::pin(val));
         }
 

martin/src/file_config.rs

@@ -140,11 +140,24 @@ pub enum FileConfigSrc {
 }
 
 impl FileConfigSrc {
-    pub fn abs_path(&self) -> Result<PathBuf, FileError> {
-        let path = match self {
+    #[must_use]
+    pub fn into_path(self) -> PathBuf {
+        match self {
+            Self::Path(p) => p,
+            Self::Obj(o) => o.path,
+        }
+    }
+
+    #[must_use]
+    pub fn get_path(&self) -> &PathBuf {
+        match self {
             Self::Path(p) => p,
             Self::Obj(o) => &o.path,
-        };
+        }
+    }
+
+    pub fn abs_path(&self) -> Result<PathBuf, FileError> {
+        let path = self.get_path();
         path.canonicalize().map_err(|e| IoError(e, path.clone()))
     }
 }
@@ -158,12 +171,12 @@ pub async fn resolve_files<Fut>(
     config: &mut FileConfigEnum,
     idr: IdResolver,
     extension: &str,
-    create_source: &mut impl FnMut(String, PathBuf) -> Fut,
+    new_source: &mut impl FnMut(String, PathBuf) -> Fut,
 ) -> Result<TileInfoSources, Error>
 where
     Fut: Future<Output = Result<Box<dyn Source>, FileError>>,
 {
-    resolve_int(config, idr, extension, create_source)
+    resolve_int(config, idr, extension, new_source)
         .map_err(crate::Error::from)
         .await
 }
@@ -172,7 +185,7 @@ async fn resolve_int<Fut>(
     config: &mut FileConfigEnum,
     idr: IdResolver,
     extension: &str,
-    create_source: &mut impl FnMut(String, PathBuf) -> Fut,
+    new_source: &mut impl FnMut(String, PathBuf) -> Fut,
 ) -> Result<TileInfoSources, FileError>
 where
     Fut: Future<Output = Result<Box<dyn Source>, FileError>>,
@@ -204,7 +217,7 @@ where
                 FileConfigSrc::Obj(pmt) => pmt.path,
                 FileConfigSrc::Path(path) => path,
             };
-            results.push(create_source(id, path).await?);
+            results.push(new_source(id, path).await?);
         }
     }
 
@@ -248,7 +261,7 @@ where
                 FileConfigSrc::Obj(pmt) => pmt.path,
                 FileConfigSrc::Path(path) => path,
             };
-            results.push(create_source(id, path).await?);
+            results.push(new_source(id, path).await?);
         }
     }
 
@@ -288,7 +301,7 @@ mod tests {
             paths,
             vec![
                 PathBuf::from("/dir-path"),
-                PathBuf::from("/path/to/file2.ext")
+                PathBuf::from("/path/to/file2.ext"),
             ]
         );
         assert_eq!(

tests/config.yaml

@@ -163,6 +163,7 @@ postgres:
 pmtiles:
   sources:
     pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
+    pmt2: tests/fixtures/pmtiles2/webp2.pmtiles
 
 sprites:
   paths: tests/fixtures/sprites/src1

tests/expected/auto/webp2_1_0_0.png.txt

@@ -0,0 +1 @@
+tests/output/auto/webp2_1_0_0.png: RIFF (little-endian) data, Web/P image, VP8 encoding, 512x512, Scaling: [none]x[none], YUV color, decoders should clamp

tests/expected/configured/catalog_cfg.json

@@ -23,6 +23,10 @@
     "pmt": {
       "content_type": "image/png"
     },
+    "pmt2": {
+      "content_type": "image/webp",
+      "name": "ne2sr"
+    },
     "points1": {
       "content_type": "application/x-protobuf",
       "description": "public.points1.geom"

tests/expected/configured/pmt2_0_0_0.png.txt

@@ -0,0 +1 @@
+tests/output/configured/pmt2_0_0_0.png: RIFF (little-endian) data, Web/P image

tests/expected/generated_config.yaml

@@ -191,14 +191,16 @@ pmtiles:
   paths:
   - tests/fixtures/mbtiles
   - tests/fixtures/pmtiles
+  - tests/fixtures/pmtiles2
   sources:
     png: tests/fixtures/pmtiles/png.pmtiles
     stamen_toner__raster_CC-BY-ODbL_z3: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
-    webp2: tests/fixtures/pmtiles/webp2.pmtiles
+    webp2: tests/fixtures/pmtiles2/webp2.pmtiles
 mbtiles:
   paths:
   - tests/fixtures/mbtiles
   - tests/fixtures/pmtiles
+  - tests/fixtures/pmtiles2
   sources:
     geography-class-jpg: tests/fixtures/mbtiles/geography-class-jpg.mbtiles
     geography-class-jpg-diff: tests/fixtures/mbtiles/geography-class-jpg-diff.mbtiles

tests/expected/given_config.yaml

@@ -160,6 +160,7 @@ postgres:
 pmtiles:
   sources:
     pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles
+    pmt2: tests/fixtures/pmtiles2/webp2.pmtiles
 sprites:
   paths: tests/fixtures/sprites/src1
   sources:

tests/test.sh

@@ -175,7 +175,7 @@ echo "Test auto configured Martin"
 TEST_OUT_DIR="$(dirname "$0")/output/auto"
 mkdir -p "$TEST_OUT_DIR"
 
-ARG=(--default-srid 900913 --auto-bounds calc --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts)
+ARG=(--default-srid 900913 --auto-bounds calc --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2 --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts)
 set -x
 $MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${LOG_DIR}/test_log_1.txt" &
 MARTIN_PROC_ID=`jobs -p | tail -n 1`
@@ -237,8 +237,9 @@ test_jsn points3857_srid          points3857
 test_pbf points3857_srid_0_0_0    points3857/0/0/0
 
 >&2 echo "***** Test server response for PMTiles source *****"
-test_jsn pmt       stamen_toner__raster_CC-BY-ODbL_z3
-test_png pmt_3_4_2 stamen_toner__raster_CC-BY-ODbL_z3/3/4/2
+test_jsn pmt         stamen_toner__raster_CC-BY-ODbL_z3
+test_png pmt_3_4_2   stamen_toner__raster_CC-BY-ODbL_z3/3/4/2
+test_png webp2_1_0_0 webp2/1/0/0  # HTTP pmtiles
 
 >&2 echo "***** Test server response for MbTiles source *****"
 test_jsn mb_jpg       geography-class-jpg
@@ -276,6 +277,7 @@ test_pbf cmp_0_0_0   points1,points2/0/0/0
 test_pbf fnc_0_0_0   function_zxy_query/0/0/0
 test_pbf fnc2_0_0_0  function_zxy_query_test/0/0/0?token=martin
 test_png pmt_0_0_0   pmt/0/0/0
+test_png pmt2_0_0_0  pmt2/0/0/0  # HTTP pmtiles
 
 test_jsn spr_src1     sprite/src1.json
 test_png spr_src1     sprite/src1.png