Add dynamic sprites support (#715)

Dynamically create image sprites for MapLibre rendering, given a
directory with images.

### TODO
* [x] Work with @flother to merge these PRs
  * [x] flother/spreet#59  (must have)
  * [x] flother/spreet#57
  * [x] flother/spreet#56
* [ ] flother/spreet#62 (not required but nice
to have, can upgrade later without any code changes)
* [x] Add docs to the book
* [x] Add CLI param, e.g. `--sprite <dir_path>`
* [x] Don't output `.sprites` in auto-genned config when not in use

### API
Per [MapLibre sprites
API](https://maplibre.org/maplibre-style-spec/sprite/), we need to
support the following:
* `/sprite/<sprite_id>.json` metadata about the sprite file - all coming
from a single directory
* `/sprite/<sprite_id>.png` all images combined into a single PNG
* `/sprite/<sprite_id>@2x.json` same but for high DPI devices
* `/sprite/<sprite_id>@2x.png`

Multiple sprite_id values can be combined into one sprite with the same
pattern as for tile joining:
`/sprite/<sprite_id1>,<sprite_id2>,...,<sprite_idN>[.json|.png|@2x.json|@2x.png]`.
No ID renaming is done, so identical names will override one another.

### Configuration
[Config file](https://maplibre.org/martin/config-file.html) and possibly
CLI should have a simple option to serve sprites. The configuration may
look similar to how mbtiles and pmtiles are configured:

```yaml
# Publish sprite images
sprites:
  paths:
    # scan this whole dir, matching all image files, and publishing it as "my_images" sprite source
    - /path/to/my_images
  sources:
    # named source matching source name to a directory
    my_sprites: /path/to/some_dir
```

Implement #705

Cargo.toml

@@ -41,6 +41,7 @@ semver = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
+spreet = { version = "0.8", default-features = false }
 sqlx = { version = "0.6", features = ["offline", "sqlite", "runtime-actix-native-tls"] }
 subst = { version = "0.2", features = ["yaml"] }
 thiserror = "1"

README.md

@@ -93,6 +93,7 @@ Martin data is available via the HTTP `GET` endpoints:
 | `/{sourceID}/{z}/{x}/{y}`              | Map Tiles                                     |
 | `/{source1},...,{sourceN}`             | Composite Source TileJSON                     |
 | `/{source1},...,{sourceN}/{z}/{x}/{y}` | Composite Source Tiles                        |
+| `/sprite/{spriteID}[@2x].{json,png}`   | Sprites (low and high DPI, index/png)         |
 | `/health`                              | Martin server health check: returns 200 `OK`  |
 
 ## Documentation

docs/src/SUMMARY.md

@@ -13,6 +13,7 @@
   - [PostgreSQL Function Sources](sources-pg-functions.md)
   - [MBTiles and PMTiles File Sources](sources-files.md)
   - [Composite Sources](sources-composite.md)
+  - [Sprite Sources](sources-sprites.md)
 - [Usage and Endpoint API](using.md)
   - [Using with MapLibre](using-with-maplibre.md)
   - [Using with Leaflet](using-with-leaflet.md)

docs/src/config-file.md

@@ -101,7 +101,7 @@ postgres:
       # covered by all zoom levels. The bounds are represented in WGS:84
       # latitude and longitude values, in the order left, bottom, right, top.
       # Values may be integers or floating point numbers.
-      bounds: [ -180.0, -90.0, 180.0, 90.0 ]
+      bounds: [-180.0, -90.0, 180.0, 90.0]
 
       # Tile extent in tile coordinate space
       extent: 4096
@@ -138,7 +138,7 @@ postgres:
       # covered by all zoom levels. The bounds are represented in WGS:84
       # latitude and longitude values, in the order left, bottom, right, top.
       # Values may be integers or floating point numbers.
-      bounds: [ -180.0, -90.0, 180.0, 90.0 ]
+      bounds: [-180.0, -90.0, 180.0, 90.0]
 
 # Publish PMTiles files
 pmtiles:
@@ -161,4 +161,13 @@ mbtiles:
   sources:
     # named source matching source name to a single file
     mb-src1: /path/to/mbtiles1.mbtiles
+
+# Sprite configuration
+sprites:
+  paths:
+    # all SVG files in this dir will be published as a "my_images" sprite source
+    - /path/to/my_images
+  sources:
+    # SVG images in this directory will be published as a "my_sprites" sprite source
+    my_sprites: /path/to/some_dir
 ```

docs/src/run-with-cli.md

@@ -13,18 +13,20 @@ Options:
           Path to config file. If set, no tile source-related parameters are allowed
       --save-config <SAVE_CONFIG>
           Save resulting config to a file or use "-" to print to stdout. By default, only print if sources are auto-detected
+  -s, --sprite <SPRITE>
+          Export a directory with SVG files as a sprite source. Can be specified multiple times
   -k, --keep-alive <KEEP_ALIVE>
           Connection keep alive timeout. [DEFAULT: 75]
   -l, --listen-addresses <LISTEN_ADDRESSES>
           The socket address to bind. [DEFAULT: 0.0.0.0:3000]
   -W, --workers <WORKERS>
           Number of web server workers
   -b, --disable-bounds
-          Disable the automatic generation of bounds for spatial tables
+          Disable the automatic generation of bounds for spatial PG tables
       --ca-root-file <CA_ROOT_FILE>
           Loads trusted root certificates from a file. The file should contain a sequence of PEM-formatted CA certificates
   -d, --default-srid <DEFAULT_SRID>
-          If a spatial table has SRID 0, then this default SRID will be used as a fallback
+          If a spatial PG table has SRID 0, then this default SRID will be used as a fallback
   -p, --pool-size <POOL_SIZE>
           Maximum connections pool size [DEFAULT: 20]
   -m, --max-feature-count <MAX_FEATURE_COUNT>

docs/src/sources-sprites.md

@@ -0,0 +1,53 @@
+# Sprite Sources
+
+Given a directory with SVG images, Martin can generate a Sprite index an a PNG image, for both low and high resolution displays. The SVG filenames without extension will be used as the sprite image IDs. The images are searched recursively in the given directory, so subdirectory names will be used as prefixes for the image IDs, e.g. `icons/bicycle.svg` will be available as `icons/bicycle` sprite image.
+
+### API
+As described in the [MapLibre sprites API](https://maplibre.org/maplibre-style-spec/sprite/), Sprites must be accessible via support the following endpoints. The sprite image and index are generated on the fly, so if the sprite directory is updated, the changes will be reflected immediately.
+
+##### Sprite PNG
+
+![sprite](sources-sprites.png)
+
+`GET /sprite/<sprite_id>.png` endpoint contains a single PNG sprite image that combines all sources images. Additionally, there is a high DPI version available at `GET /sprite/<sprite_id>@2x.png`.
+
+##### Sprite index
+`/sprite/<sprite_id>.json` metadata index describing the position and size of each image inside the sprite. Just like the PNG, there is a high DPI version available at `/sprite/<sprite_id>@2x.json`.
+
+```json
+{
+  "bicycle": {
+    "height": 15,
+    "pixelRatio": 1,
+    "width": 15,
+    "x": 20,
+    "y": 16
+  },
+  ...
+}
+```
+#### Combining Multiple Sprites
+Multiple sprite_id values can be combined into one sprite with the same pattern as for tile joining:  `/sprite/<sprite_id1>,<sprite_id2>,...,<sprite_idN>`. No ID renaming is done, so identical sprite names will override one another.
+
+### Configuring from CLI
+
+A sprite directory can be configured from the CLI with the `--sprite` flag. The flag can be used multiple times to configure multiple sprite directories. The name of the sprite will be the name of the directory -- in the example below, the sprites will be available at `/sprite/sprite_a` and `/sprite/sprite_b`.  Use `--save-config` to save the configuration to the config file.
+
+```shell
+martin --sprite /path/to/sprite_a --sprite /path/to/other/sprite_b
+```
+
+### Configuring with Config File
+
+A sprite directory can be configured from the config file with the `sprite` key, similar to how [MBTiles and PMTiles](config-file.md) are configured.
+
+```yaml
+# Sprite configuration
+sprites:
+  paths:
+    # all SVG files in this dir will be published as a "my_images" sprite source
+    - /path/to/my_images
+  sources:
+    # SVG images in this directory will be published as a "my_sprites" sprite source
+    my_sprites: /path/to/some_dir
+```

docs/src/using.md

@@ -10,6 +10,7 @@ Martin data is available via the HTTP `GET` endpoints:
 | `/{sourceID}/{z}/{x}/{y}`              | Map Tiles                                      |
 | `/{source1},...,{sourceN}`             | [Composite Source TileJSON](#source-tilejson)  |
 | `/{source1},...,{sourceN}/{z}/{x}/{y}` | [Composite Source Tiles](sources-composite.md) |
+| `/sprite/{spriteID}[@2x].{json,png}`   | [Sprite sources](sources-sprites.md)           |
 | `/health`                              | Martin server health check: returns 200 `OK`   |
 
 ## Duplicate Source ID

justfile

@@ -106,6 +106,7 @@ test-int: clean-test
 
 # Run integration tests and save its output as the new expected output
 bless: start clean-test
+    cargo test --features bless-tests
     tests/test.sh
     rm -rf tests/expected
     mv tests/output tests/expected

martin/Cargo.toml

@@ -25,6 +25,7 @@ path = "src/bin/main.rs"
 default = []
 ssl = ["openssl", "postgres-openssl"]
 vendored-openssl = ["ssl", "openssl/vendored"]
+bless-tests = []
 
 [dependencies]
 actix-cors.workspace = true
@@ -53,9 +54,11 @@ semver.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
 serde_yaml.workspace = true
+spreet.workspace = true
 subst.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
+tokio = { workspace = true, features = ["io-std"] }
 
 # Optional dependencies for openssl support
 openssl = { workspace = true, optional = true }

martin/src/args/root.rs

@@ -41,6 +41,9 @@ pub struct MetaArgs {
     pub watch: bool,
     /// Connection strings, e.g. postgres://... or /path/to/files
     pub connection: Vec<String>,
+    /// Export a directory with SVG files as a sprite source. Can be specified multiple times.
+    #[arg(short, long)]
+    pub sprite: Vec<PathBuf>,
 }
 
 impl Args {
@@ -74,6 +77,10 @@ impl Args {
             config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles");
         }
 
+        if !self.meta.sprite.is_empty() {
+            config.sprites = FileConfigEnum::new(self.meta.sprite);
+        }
+
         cli_strings.check()
     }
 }
@@ -92,11 +99,7 @@ pub fn parse_file_args(cli_strings: &mut Arguments, extension: &str) -> Option<F
         Err(_) => Ignore,
     });
 
-    match paths.len() {
-        0 => None,
-        1 => Some(FileConfigEnum::Path(paths.into_iter().next().unwrap())),
-        _ => Some(FileConfigEnum::Paths(paths)),
-    }
+    FileConfigEnum::new(paths)
 }
 
 #[cfg(test)]

martin/src/config.rs

@@ -7,20 +7,23 @@ use std::pin::Pin;
 
 use futures::future::try_join_all;
 use serde::{Deserialize, Serialize};
-use serde_yaml::Value;
 use subst::VariableMap;
 
 use crate::file_config::{resolve_files, FileConfigEnum};
 use crate::mbtiles::MbtSource;
 use crate::pg::PgConfig;
 use crate::pmtiles::PmtSource;
 use crate::source::Sources;
+use crate::sprites::{resolve_sprites, SpriteSources};
 use crate::srv::SrvConfig;
 use crate::utils::{IdResolver, OneOrMany, Result};
 use crate::Error::{ConfigLoadError, ConfigParseError, NoSources};
 
+pub type UnrecognizedValues = HashMap<String, serde_yaml::Value>;
+
 pub struct AllSources {
     pub sources: Sources,
+    pub sprites: SpriteSources,
 }
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -37,14 +40,17 @@ pub struct Config {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mbtiles: Option<FileConfigEnum>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sprites: Option<FileConfigEnum>,
+
     #[serde(flatten)]
-    pub unrecognized: HashMap<String, Value>,
+    pub unrecognized: UnrecognizedValues,
 }
 
 impl Config {
     /// Apply defaults to the config, and validate if there is a connection string
-    pub fn finalize(&mut self) -> Result<Unrecognized> {
-        let mut res = Unrecognized::new();
+    pub fn finalize(&mut self) -> Result<UnrecognizedValues> {
+        let mut res = UnrecognizedValues::new();
         copy_unrecognized_config(&mut res, "", &self.unrecognized);
 
         let mut any = if let Some(pg) = &mut self.postgres {
@@ -70,6 +76,13 @@ impl Config {
             false
         };
 
+        any |= if let Some(cfg) = &mut self.sprites {
+            res.extend(cfg.finalize("sprites.")?);
+            !cfg.is_empty()
+        } else {
+            false
+        };
+
         if any {
             Ok(res)
         } else {
@@ -87,13 +100,13 @@ impl Config {
                 sources.push(Box::pin(s.resolve(idr.clone())));
             }
         }
-        if let Some(v) = self.pmtiles.as_mut() {
-            let val = resolve_files(v, idr.clone(), "pmtiles", create_pmt_src);
+        if self.pmtiles.is_some() {
+            let val = resolve_files(&mut self.pmtiles, idr.clone(), "pmtiles", create_pmt_src);
             sources.push(Box::pin(val));
         }
 
-        if let Some(v) = self.mbtiles.as_mut() {
-            let val = resolve_files(v, idr.clone(), "mbtiles", create_mbt_src);
+        if self.mbtiles.is_some() {
+            let val = resolve_files(&mut self.mbtiles, idr.clone(), "mbtiles", create_mbt_src);
             sources.push(Box::pin(val));
         }
 
@@ -105,16 +118,15 @@ impl Config {
                     acc
                 },
             ),
+            sprites: resolve_sprites(&mut self.sprites)?,
         })
     }
 }
 
-pub type Unrecognized = HashMap<String, Value>;
-
 pub fn copy_unrecognized_config(
-    result: &mut Unrecognized,
+    result: &mut UnrecognizedValues,
     prefix: &str,
-    unrecognized: &Unrecognized,
+    unrecognized: &UnrecognizedValues,
 ) {
     result.extend(
         unrecognized