PMTiles cache, refactor file configs, modularize (#1094)

* Make it possible to have configuration specific to the file-based
config sections: pmtiles, mbtiles, and sprites.
* Implement PMTiles directory cache shared between all pmtiles (both
http and local), with configurable max cache size (in MB), or 0 to
disable. Defaults to 32MB (?)
* PMTiles now share web client instance, which optimizes connection
reuse in case multiple pmtiles reside on the same host
* Major refactoring to allow modular reuse, enabling the following build
features:
    * **postgres** - enable PostgreSQL/PostGIS tile sources
    * **pmtiles** - enable PMTile tile sources
    * **mbtiles** - enable MBTile tile sources
    * **fonts** - enable font sources
    * **sprites** - enable sprite sources
* Use justfile in the CI

Fixes #1093

Cargo.toml

@@ -48,8 +48,8 @@ insta = "1"
 itertools = "0.12"
 json-patch = "1.2"
 log = "0.4"
-martin-tile-utils = { path = "./martin-tile-utils", version = "0.3.0" }
-mbtiles = { path = "./mbtiles", version = "0.8.0" }
+martin-tile-utils = { path = "./martin-tile-utils", version = "0.4.0" }
+mbtiles = { path = "./mbtiles", version = "0.9.0" }
 moka = { version = "0.12", features = ["future"] }
 num_cpus = "1"
 pbf_font_tools = { version = "2.5.0", features = ["freetype"] }

README.md

@@ -106,6 +106,16 @@ Martin data is available via the HTTP `GET` endpoints:
 | `/font/{font1},…,{fontN}/{start}-{end}` | Composite Font source                         |
 | `/health`                               | Martin server health check: returns 200 `OK`  |
 
+## Re-use Martin as a library
+
+Martin can be used as a standalone server, or as a library in your own Rust application. When used as a library, you can use the following features:
+
+* **postgres** - enable PostgreSQL/PostGIS tile sources
+* **pmtiles** - enable PMTile tile sources
+* **mbtiles** - enable MBTile tile sources
+* **fonts** - enable font sources
+* **sprites** - enable sprite sources
+
 ## Documentation
 
 See [Martin book](https://maplibre.org/martin/) for complete documentation.

debian/config.yaml

@@ -17,6 +17,7 @@ worker_processes: 8
 #   auto_bounds: skip
 
 # pmtiles:
+#   dir_cache_size_mb: 100
 #   paths:
 #     - /dir-path
 #     - /path/to/pmtiles.pmtiles
@@ -32,6 +33,9 @@ worker_processes: 8
 #   sources:
 #     mb-src1: /path/to/mbtiles1.mbtiles
 
+# sprites:
+#   - /path/to/sprites_dir
+
 # fonts:
 #   - /path/to/font/file.ttf
 #   - /path/to/font_dir

docs/src/config-file.md

@@ -155,6 +155,8 @@ postgres:
 
 # Publish PMTiles files from local disk or proxy to a web server
 pmtiles:
+  # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] 
+  dir_cache_size_mb: 100
   paths:
     # scan this whole dir, matching all *.pmtiles files
     - /dir-path

justfile

@@ -273,7 +273,16 @@ fmt2:
 
 # Run cargo check
 check:
-    cargo check --workspace --all-targets --bins --tests --lib --benches
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin-tile-utils
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p mbtiles
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p mbtiles --no-default-features
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features fonts
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features mbtiles
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features pmtiles
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features postgres
+    RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features sprites
 
 # Verify doc build
 check-doc:
@@ -289,7 +298,7 @@ clippy-md:
       'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")" | tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}'
 
 # These steps automatically run before git push via a git hook
-git-pre-push: env-info restart fmt clippy check check-doc test
+git-pre-push: env-info restart fmt clippy check-doc test check
 
 # Get environment info
 [private]

martin-tile-utils/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "martin-tile-utils"
-version = "0.3.1"
+version = "0.4.0"
 authors = ["Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server."
 keywords = ["maps", "tiles", "mvt", "tileserver"]

martin/Cargo.toml

@@ -3,7 +3,7 @@ lints.workspace = true
 [package]
 name = "martin"
 # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin
-version = "0.11.6"
+version = "0.12.0"
 authors = ["Stepan Kuzmin <[email protected]>", "Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support"
 keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"]
@@ -59,9 +59,12 @@ name = "bench"
 harness = false
 
 [features]
-default = ["sprites", "fonts"]
-sprites = []
-fonts = []
+default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"]
+fonts = ["dep:bit-set","dep:pbf_font_tools"]
+mbtiles = []
+pmtiles = ["dep:moka"]
+postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"]
+sprites = ["dep:spreet"]
 bless-tests = []
 
 [dependencies]
@@ -70,41 +73,41 @@ actix-http.workspace = true
 actix-rt.workspace = true
 actix-web.workspace = true
 async-trait.workspace = true
-bit-set.workspace = true
+bit-set = { workspace = true, optional = true }
 brotli.workspace = true
 clap.workspace = true
-deadpool-postgres.workspace = true
+deadpool-postgres = { workspace = true, optional = true }
 env_logger.workspace = true
 flate2.workspace = true
 futures.workspace = true
 itertools.workspace = true
-json-patch.workspace = true
+json-patch = { workspace = true, optional = true }
 log.workspace = true
 martin-tile-utils.workspace = true
 mbtiles.workspace = true
-moka.workspace = true
+moka = { workspace = true, optional = true }
 num_cpus.workspace = true
-pbf_font_tools.workspace = true
+pbf_font_tools = { workspace = true, optional = true }
 pmtiles.workspace = true
-postgis.workspace = true
-postgres-protocol.workspace = true
-postgres.workspace = true
+postgis = { workspace = true, optional = true }
+postgres-protocol = { workspace = true, optional = true }
+postgres = { workspace = true, optional = true }
 regex.workspace = true
 reqwest.workspace = true
 rustls-native-certs.workspace = true
 rustls-pemfile.workspace = true
 rustls.workspace = true
-semver.workspace = true
+semver = { workspace = true, optional = true }
 serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 serde_yaml.workspace = true
-spreet.workspace = true
+spreet = { workspace = true, optional = true }
 subst.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
 tokio = { workspace = true, features = ["io-std"] }
-tokio-postgres-rustls.workspace = true
+tokio-postgres-rustls = { workspace = true, optional = true }
 url.workspace = true
 
 [dev-dependencies]

martin/src/args/mod.rs

@@ -4,7 +4,9 @@ pub use connections::{Arguments, State};
 mod environment;
 pub use environment::{Env, OsEnv};
 
+#[cfg(feature = "postgres")]
 mod pg;
+#[cfg(feature = "postgres")]
 pub use pg::{BoundsCalcType, PgArgs, DEFAULT_BOUNDS_TIMEOUT};
 
 mod root;

martin/src/args/root.rs

@@ -2,14 +2,12 @@ use std::path::PathBuf;
 
 use clap::Parser;
 use log::warn;
-use url::Url;
 
 use crate::args::connections::Arguments;
 use crate::args::environment::Env;
-use crate::args::pg::PgArgs;
 use crate::args::srv::SrvArgs;
-use crate::args::State::{Ignore, Share, Take};
 use crate::config::Config;
+#[cfg(any(feature = "mbtiles", feature = "pmtiles", feature = "sprites"))]
 use crate::file_config::FileConfigEnum;
 use crate::MartinError::ConfigAndConnectionsError;
 use crate::{MartinResult, OptOneMany};
@@ -27,8 +25,9 @@ pub struct Args {
     pub extras: ExtraArgs,
     #[command(flatten)]
     pub srv: SrvArgs,
+    #[cfg(feature = "postgres")]
     #[command(flatten)]
-    pub pg: Option<PgArgs>,
+    pub pg: Option<crate::args::pg::PgArgs>,
 }
 
 // None of these params will be transferred to the config
@@ -80,19 +79,26 @@ impl Args {
 
         self.srv.merge_into_config(&mut config.srv);
 
+        #[allow(unused_mut)]
         let mut cli_strings = Arguments::new(self.meta.connection);
-        let pg_args = self.pg.unwrap_or_default();
-        if config.postgres.is_none() {
-            config.postgres = pg_args.into_config(&mut cli_strings, env);
-        } else {
-            // config was loaded from a file, we can only apply a few CLI overrides to it
-            pg_args.override_config(&mut config.postgres, env);
+
+        #[cfg(feature = "postgres")]
+        {
+            let pg_args = self.pg.unwrap_or_default();
+            if config.postgres.is_none() {
+                config.postgres = pg_args.into_config(&mut cli_strings, env);
+            } else {
+                // config was loaded from a file, we can only apply a few CLI overrides to it
+                pg_args.override_config(&mut config.postgres, env);
+            }
         }
 
+        #[cfg(feature = "pmtiles")]
         if !cli_strings.is_empty() {
             config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles", true);
         }
 
+        #[cfg(feature = "mbtiles")]
         if !cli_strings.is_empty() {
             config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles", false);
         }
@@ -110,9 +116,10 @@ impl Args {
     }
 }
 
+#[cfg(any(feature = "pmtiles", feature = "mbtiles"))]
 fn is_url(s: &str, extension: &str) -> bool {
     if s.starts_with("http") {
-        if let Ok(url) = Url::parse(s) {
+        if let Ok(url) = url::Url::parse(s) {
             if url.scheme() == "http" || url.scheme() == "https" {
                 if let Some(ext) = url.path().rsplit('.').next() {
                     return ext == extension;
@@ -123,11 +130,14 @@ fn is_url(s: &str, extension: &str) -> bool {
     false
 }
 
-pub fn parse_file_args(
+#[cfg(any(feature = "pmtiles", feature = "mbtiles"))]
+pub fn parse_file_args<T: crate::file_config::ConfigExtras>(
     cli_strings: &mut Arguments,
     extension: &str,
     allow_url: bool,
-) -> FileConfigEnum {
+) -> FileConfigEnum<T> {
+    use crate::args::State::{Ignore, Share, Take};
+
     let paths = cli_strings.process(|s| match PathBuf::try_from(s) {
         Ok(v) => {
             if allow_url && is_url(s, extension) {
@@ -149,9 +159,7 @@ pub fn parse_file_args(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::pg::PgConfig;
-    use crate::test_utils::{some, FauxEnv};
-    use crate::utils::OptOneMany;
+    use crate::test_utils::FauxEnv;
     use crate::MartinError::UnrecognizableConnections;
 
     fn parse(args: &[&str]) -> MartinResult<(Config, MetaArgs)> {
@@ -169,8 +177,12 @@ mod tests {
         assert_eq!(args, expected);
     }
 
+    #[cfg(feature = "postgres")]
     #[test]
     fn cli_with_config() {
+        use crate::test_utils::some;
+        use crate::utils::OptOneMany;
+
         let args = parse(&["martin", "--config", "c.toml"]).unwrap();
         let meta = MetaArgs {
             config: Some(PathBuf::from("c.toml")),
@@ -188,7 +200,7 @@ mod tests {
 
         let args = parse(&["martin", "postgres://connection"]).unwrap();
         let cfg = Config {
-            postgres: OptOneMany::One(PgConfig {
+            postgres: OptOneMany::One(crate::pg::PgConfig {
                 connection_string: some("postgres://connection"),
                 ..Default::default()
             }),

martin/src/bin/martin-cp.rs

@@ -11,7 +11,7 @@ use clap::Parser;
 use futures::stream::{self, StreamExt};
 use futures::TryStreamExt;
 use log::{debug, error, info, log_enabled};
-use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, PgArgs, SrvArgs};
+use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs};
 use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS};
 use martin::{
     append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source,
@@ -46,8 +46,9 @@ pub struct CopierArgs {
     pub copy: CopyArgs,
     #[command(flatten)]
     pub meta: MetaArgs,
+    #[cfg(feature = "postgres")]
     #[command(flatten)]
-    pub pg: Option<PgArgs>,
+    pub pg: Option<martin::args::PgArgs>,
 }
 
 #[serde_with::serde_as]
@@ -137,6 +138,7 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> {
         meta: copy_args.meta,
         extras: ExtraArgs::default(),
         srv: SrvArgs::default(),
+        #[cfg(feature = "postgres")]
         pg: copy_args.pg,
     };