Implement sprite support

Serve image assets sprites for MapLibre rendering, given a directory with images.

### API
Per [MapLibre sprites API](https://maplibre.org/maplibre-style-spec/sprite/), we need to support the following:
* `/sprite/<sprite_id>.json` metadata about the sprite file - all coming from a single directory
* `/sprite/<sprite_id>.png` all images combined into a single PNG
* `/sprite/<sprite_id>@2x.json` same but for high DPI devices
* `/sprite/<sprite_id>@2x.png`

Multiple sprite_id values can be combined into one sprite with the same pattern as for tile joining:  `/sprite/<sprite_id1>,<sprite_id2>,...,<sprite_idN>[.json|.png|@2x.json|@2x.png]`. No ID renaming is done, so identical names will override one another.

### Configuration
[Config file](https://maplibre.org/martin/config-file.html) and possibly CLI should have a simple option to serve sprites.  The configuration may look similar to how mbtiles and pmtiles are configured:

```yaml
# Publish sprite images
sprites:
  paths:
    # scan this whole dir, matching all image files, and publishing it as "my_images" sprite source
    - /path/to/my_images
  sources:
    # named source matching source name to a directory
    my_sprites: /path/to/some_dir
```

Implement maplibre#705

Cargo.toml

@@ -36,11 +36,13 @@ postgis = "0.9"
 postgres = { version = "0.19", features = ["with-time-0_3", "with-uuid-1", "with-serde_json-1"] }
 postgres-openssl = "0.5"
 postgres-protocol = "0.6"
+rayon = "1.7"
 regex = "1"
 semver = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
+spreet = { version = "0.8.0-dev", path = "../spreet" }
 sqlx = { version = "0.6", features = ["offline", "sqlite", "runtime-actix-native-tls"] }
 subst = { version = "0.2", features = ["yaml"] }
 thiserror = "1"

README.md

@@ -93,6 +93,7 @@ Martin data is available via the HTTP `GET` endpoints:
 | `/{sourceID}/{z}/{x}/{y}`              | Map Tiles                                     |
 | `/{source1},...,{sourceN}`             | Composite Source TileJSON                     |
 | `/{source1},...,{sourceN}/{z}/{x}/{y}` | Composite Source Tiles                        |
+| `/sprite/{spriteID}[@2x].{json,png}`   | Sprites (low and high DPI, index/png)         |
 | `/health`                              | Martin server health check: returns 200 `OK`  |
 
 ## Documentation

justfile

@@ -106,6 +106,7 @@ test-int: clean-test
 
 # Run integration tests and save its output as the new expected output
 bless: start clean-test
+    cargo test --features bless-tests
     tests/test.sh
     rm -rf tests/expected
     mv tests/output tests/expected

martin/Cargo.toml

@@ -25,6 +25,7 @@ path = "src/bin/main.rs"
 default = []
 ssl = ["openssl", "postgres-openssl"]
 vendored-openssl = ["ssl", "openssl/vendored"]
+bless-tests = []
 
 [dependencies]
 actix-cors.workspace = true
@@ -53,9 +54,11 @@ semver.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
 serde_yaml.workspace = true
+spreet.workspace = true
 subst.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
+tokio = { workspace = true, features = ["io-std"] }
 
 # Optional dependencies for openssl support
 openssl = { workspace = true, optional = true }

martin/src/config.rs

@@ -14,10 +14,14 @@ use crate::file_config::{resolve_files, FileConfigEnum};
 use crate::mbtiles::MbtSource;
 use crate::pg::PgConfig;
 use crate::pmtiles::PmtSource;
-use crate::source::{IdResolver, Sources};
+use crate::source::Sources;
+use crate::sprites::{resolve_sprites, SpriteSources};
 use crate::srv::SrvConfig;
 use crate::utils::{OneOrMany, Result};
 use crate::Error::{ConfigLoadError, ConfigParseError, NoSources};
+use crate::IdResolver;
+
+pub type SourceTypes = (Sources, SpriteSources);
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct Config {
@@ -33,6 +37,9 @@ pub struct Config {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mbtiles: Option<FileConfigEnum>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sprites: Option<FileConfigEnum>,
+
     #[serde(flatten)]
     pub unrecognized: HashMap<String, Value>,
 }
@@ -73,7 +80,7 @@ impl Config {
         }
     }
 
-    pub async fn resolve(&mut self, idr: IdResolver) -> Result<Sources> {
+    pub async fn resolve(&mut self, idr: IdResolver) -> Result<SourceTypes> {
         let create_pmt_src = &mut PmtSource::new_box;
         let create_mbt_src = &mut MbtSource::new_box;
 
@@ -93,13 +100,16 @@ impl Config {
             sources.push(Box::pin(val));
         }
 
-        Ok(try_join_all(sources)
-            .await?
-            .into_iter()
-            .fold(HashMap::new(), |mut acc, hashmap| {
-                acc.extend(hashmap);
-                acc
-            }))
+        Ok((
+            try_join_all(sources)
+                .await?
+                .into_iter()
+                .fold(HashMap::new(), |mut acc, hashmap| {
+                    acc.extend(hashmap);
+                    acc
+                }),
+            resolve_sprites(&mut self.sprites)?,
+        ))
     }
 }
 

martin/src/file_config.rs

@@ -40,6 +40,22 @@ pub enum FileConfigEnum {
     Config(FileConfig),
 }
 
+impl FileConfigEnum {
+    pub fn extract_file_config(&mut self) -> FileConfig {
+        match self {
+            FileConfigEnum::Path(path) => FileConfig {
+                paths: Some(One(mem::take(path))),
+                ..FileConfig::default()
+            },
+            FileConfigEnum::Paths(paths) => FileConfig {
+                paths: Some(Many(mem::take(paths))),
+                ..Default::default()
+            },
+            FileConfigEnum::Config(cfg) => mem::take(cfg),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct FileConfig {
     /// A list of file paths
@@ -69,12 +85,12 @@ pub enum FileConfigSrc {
 }
 
 impl FileConfigSrc {
-    #[must_use]
-    pub fn path(&self) -> &PathBuf {
-        match self {
+    pub fn path(&self) -> Result<PathBuf, FileError> {
+        let path = match self {
             Self::Path(p) => p,
             Self::Obj(o) => &o.path,
-        }
+        };
+        path.canonicalize().map_err(|e| IoError(e, path.clone()))
     }
 }
 
@@ -125,17 +141,7 @@ async fn resolve_int<Fut>(
 where
     Fut: Future<Output = Result<Box<dyn Source>, FileError>>,
 {
-    let cfg = match config {
-        FileConfigEnum::Path(path) => FileConfig {
-            paths: Some(One(mem::take(path))),
-            ..FileConfig::default()
-        },
-        FileConfigEnum::Paths(paths) => FileConfig {
-            paths: Some(Many(mem::take(paths))),
-            ..Default::default()
-        },
-        FileConfigEnum::Config(cfg) => mem::take(cfg),
-    };
+    let cfg = config.extract_file_config();
 
     let mut results = Sources::new();
     let mut configs = HashMap::new();
@@ -144,8 +150,7 @@ where
 
     if let Some(sources) = cfg.sources {
         for (id, source) in sources {
-            let path = source.path();
-            let can = path.canonicalize().map_err(|e| IoError(e, path.clone()))?;
+            let can = source.path()?;
             if !can.is_file() {
                 // todo: maybe warn instead?
                 return Err(InvalidSourceFilePath(id.to_string(), can));
@@ -173,7 +178,7 @@ where
                 directories.push(path.clone());
                 path.read_dir()
                     .map_err(|e| IoError(e, path.clone()))?
-                    .filter_map(std::result::Result::ok)
+                    .filter_map(Result::ok)
                     .filter(|f| {
                         f.path().extension().filter(|e| *e == extension).is_some()
                             && f.path().is_file()

martin/src/lib.rs

@@ -14,6 +14,7 @@ pub mod mbtiles;
 pub mod pg;
 pub mod pmtiles;
 mod source;
+pub mod sprites;
 pub mod srv;
 mod utils;
 
@@ -26,8 +27,10 @@ mod test_utils;
 #[cfg(test)]
 pub use crate::args::Env;
 pub use crate::config::{read_config, Config};
-pub use crate::source::{IdResolver, Source, Sources, Xyz};
-pub use crate::utils::{decode_brotli, decode_gzip, BoolOrObject, Error, OneOrMany, Result};
+pub use crate::source::{Source, Sources, Xyz};
+pub use crate::utils::{
+    decode_brotli, decode_gzip, BoolOrObject, Error, IdResolver, OneOrMany, Result,
+};
 
 // Ensure README.md contains valid code
 #[cfg(doctest)]

martin/src/pg/config.rs

@@ -7,8 +7,9 @@ use crate::pg::config_function::FuncInfoSources;
 use crate::pg::config_table::TableInfoSources;
 use crate::pg::configurator::PgBuilder;
 use crate::pg::utils::Result;
-use crate::source::{IdResolver, Sources};
+use crate::source::Sources;
 use crate::utils::{sorted_opt_map, BoolOrObject, OneOrMany};
+use crate::IdResolver;
 
 pub trait PgInfo {
     fn format_id(&self) -> String;

martin/src/pg/configurator.rs

@@ -1,6 +1,7 @@
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
 
+use crate::IdResolver;
 use futures::future::join_all;
 use itertools::Itertools;
 use log::{debug, error, info, warn};
@@ -14,7 +15,7 @@ use crate::pg::pool::PgPool;
 use crate::pg::table_source::{calc_srid, get_table_sources, merge_table_info, table_to_query};
 use crate::pg::utils::PgError::InvalidTableExtent;
 use crate::pg::utils::Result;
-use crate::source::{IdResolver, Sources};
+use crate::source::Sources;
 use crate::utils::{find_info, normalize_key, BoolOrObject, InfoMap, OneOrMany};
 
 pub type SqlFuncInfoMapMap = InfoMap<InfoMap<(PgSqlInfo, FunctionInfo)>>;

martin/src/source.rs

@@ -1,10 +1,7 @@
-use std::collections::hash_map::Entry;
-use std::collections::{HashMap, HashSet};
-use std::fmt::{Debug, Display, Formatter, Write};
-use std::sync::{Arc, Mutex};
-
 use async_trait::async_trait;
 use martin_tile_utils::TileInfo;
+use std::collections::HashMap;
+use std::fmt::{Debug, Display, Formatter};
 use tilejson::TileJSON;
 
 use crate::utils::Result;
@@ -51,94 +48,10 @@ impl Clone for Box<dyn Source> {
     }
 }
 
-#[derive(Debug, Default, Clone)]
-pub struct IdResolver {
-    /// name -> unique name
-    names: Arc<Mutex<HashMap<String, String>>>,
-    /// reserved names
-    reserved: HashSet<&'static str>,
-}
-
-impl IdResolver {
-    #[must_use]
-    pub fn new(reserved_keywords: &[&'static str]) -> Self {
-        Self {
-            names: Arc::new(Mutex::new(HashMap::new())),
-            reserved: reserved_keywords.iter().copied().collect(),
-        }
-    }
-
-    /// If source name already exists in the self.names structure,
-    /// try appending it with ".1", ".2", etc. until the name is unique.
-    /// Only alphanumeric characters plus dashes/dots/underscores are allowed.
-    #[must_use]
-    pub fn resolve(&self, name: &str, unique_name: String) -> String {
-        // Ensure name has no prohibited characters like spaces, commas, slashes, or non-unicode etc.
-        // Underscores, dashes, and dots are OK. All other characters will be replaced with dashes.
-        let mut name = name.replace(
-            |c: char| !c.is_ascii_alphanumeric() && c != '_' && c != '.' && c != '-',
-            "-",
-        );
-
-        let mut names = self.names.lock().expect("IdResolver panicked");
-        if !self.reserved.contains(name.as_str()) {
-            match names.entry(name) {
-                Entry::Vacant(e) => {
-                    let id = e.key().clone();
-                    e.insert(unique_name);
-                    return id;
-                }
-                Entry::Occupied(e) => {
-                    name = e.key().clone();
-                    if e.get() == &unique_name {
-                        return name;
-                    }
-                }
-            }
-        }
-        // name already exists, try it with ".1", ".2", etc. until the value matches
-        // assume that reserved keywords never end in a "dot number", so don't check
-        let mut index: i32 = 1;
-        let mut new_name = String::new();
-        loop {
-            new_name.clear();
-            write!(&mut new_name, "{name}.{index}").unwrap();
-            index = index.checked_add(1).unwrap();
-            match names.entry(new_name.clone()) {
-                Entry::Vacant(e) => {
-                    e.insert(unique_name);
-                    return new_name;
-                }
-                Entry::Occupied(e) => {
-                    if e.get() == &unique_name {
-                        return new_name;
-                    }
-                }
-            }
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    #[test]
-    fn id_resolve() {
-        let r = IdResolver::default();
-        assert_eq!(r.resolve("a", "a".to_string()), "a");
-        assert_eq!(r.resolve("a", "a".to_string()), "a");
-        assert_eq!(r.resolve("a", "b".to_string()), "a.1");
-        assert_eq!(r.resolve("a", "b".to_string()), "a.1");
-        assert_eq!(r.resolve("b", "a".to_string()), "b");
-        assert_eq!(r.resolve("b", "a".to_string()), "b");
-        assert_eq!(r.resolve("a.1", "a".to_string()), "a.1.1");
-        assert_eq!(r.resolve("a.1", "b".to_string()), "a.1");
-
-        assert_eq!(r.resolve("a b", "a b".to_string()), "a-b");
-        assert_eq!(r.resolve("a b", "ab2".to_string()), "a-b.1");
-    }
-
     #[test]
     fn xyz_format() {
         let xyz = Xyz { z: 1, x: 2, y: 3 };