From 3dc54d7f9eaac777f3d650cdb285184e51845725 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Dec 2023 22:48:23 -0500 Subject: [PATCH] Implement tile caching (#1105) Add a top level config parameter -- the size of cache memory (in MB) to use for caching tiles and PMT directories, defaulting to 512, and 0 to disable. This also removes the `pmtiles.dir_cache_size_mb` parameter (it will be ignored, but will give a warning) ``` cache_size_mb: 512 ``` The new cache will contain all tiles as provided by the source. So if PostgreSQL returns a non-compressed tile, the cache will contain the uncompressed variant, and will be compressed for each response. This will be fixed in the later releases. Note that fonts and sprites are not cached at this time, and are still a TODO. --- Cargo.lock | 40 ++-- debian/config.yaml | 4 +- docs/src/config-file.md | 5 +- martin-tile-utils/Cargo.toml | 2 +- martin-tile-utils/src/lib.rs | 2 +- martin/Cargo.toml | 6 +- martin/benches/bench.rs | 5 +- martin/src/args/root.rs | 7 + martin/src/bin/martin-cp.rs | 45 ++-- martin/src/bin/martin.rs | 6 +- martin/src/config.rs | 41 +++- martin/src/file_config.rs | 21 +- martin/src/lib.rs | 2 +- martin/src/pmtiles/mod.rs | 53 ++--- martin/src/source.rs | 1 + martin/src/sprites/mod.rs | 2 +- martin/src/srv/fonts.rs | 0 martin/src/srv/mod.rs | 8 +- martin/src/srv/server.rs | 4 +- martin/src/srv/sprites.rs | 27 +-- martin/src/srv/tiles.rs | 249 ++++++++++++--------- martin/src/srv/tiles_info.rs | 0 martin/src/utils/cache.rs | 91 ++++++++ martin/src/utils/mod.rs | 3 + martin/src/utils/xyz.rs | 2 +- martin/tests/mb_server_test.rs | 1 + martin/tests/pg_server_test.rs | 2 + martin/tests/pmt_server_test.rs | 1 + martin/tests/utils/pg_utils.rs | 4 +- mbtiles/Cargo.toml | 2 +- tests/config.yaml | 4 +- tests/expected/configured/save_config.yaml | 2 +- 32 files changed, 417 insertions(+), 225 deletions(-) mode change 100755 => 100644 martin/src/srv/fonts.rs mode change 100644 => 100755 martin/src/srv/tiles.rs mode change 100644 => 100755 martin/src/srv/tiles_info.rs create mode 100755 martin/src/utils/cache.rs diff --git a/Cargo.lock b/Cargo.lock index c939d7e31..e789b0ffe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" [[package]] name = "approx" @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", @@ -1025,9 +1025,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -1707,9 +1707,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2010,7 +2010,7 @@ dependencies = [ [[package]] name = "martin" -version = "0.12.0" +version = "0.13.0" dependencies = [ "actix-cors", "actix-http", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" dependencies = [ "approx", "insta", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix-rt", "anyhow", @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" dependencies = [ "unicode-ident", ] @@ -4004,18 +4004,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" +checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" +checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19" dependencies = [ "proc-macro2", "quote", @@ -4674,11 +4674,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] diff --git a/debian/config.yaml b/debian/config.yaml index f44db6ad2..30b9ad27e 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -7,6 +7,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 512 + # see https://maplibre.org/martin/config-file.html # postgres: @@ -17,7 +20,6 @@ worker_processes: 8 # auto_bounds: skip # pmtiles: -# dir_cache_size_mb: 100 # paths: # - /dir-path # - /path/to/pmtiles.pmtiles diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 19cfec94f..542677de1 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -24,6 +24,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 1024 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string. You can use env vars too, for example: @@ -155,8 +158,6 @@ postgres: # Publish PMTiles files from local disk or proxy to a web server pmtiles: - # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] - dir_cache_size_mb: 100 paths: # scan this whole dir, matching all *.pmtiles files - /dir-path diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index 1f6899566..90d39ce7d 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server." keywords = ["maps", "tiles", "mvt", "tileserver"] diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index 029949ebe..9df4fd498 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -86,7 +86,7 @@ impl Display for Format { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub enum Encoding { /// Data is not compressed, but it can be Uncompressed = 0b0000_0000, diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 38405a0f2..9e81bec80 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -3,7 +3,7 @@ lints.workspace = true [package] name = "martin" # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin -version = "0.12.0" +version = "0.13.0" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"] @@ -62,7 +62,7 @@ harness = false default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] fonts = ["dep:bit-set", "dep:pbf_font_tools"] mbtiles = [] -pmtiles = ["dep:moka"] +pmtiles = [] postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] sprites = ["dep:spreet"] bless-tests = [] @@ -85,7 +85,7 @@ json-patch = { workspace = true, optional = true } log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true -moka = { workspace = true, optional = true } +moka.workspace = true num_cpus.workspace = true pbf_font_tools = { workspace = true, optional = true } pmtiles.workspace = true diff --git a/martin/benches/bench.rs b/martin/benches/bench.rs index 77363284b..27f4904b2 100644 --- a/martin/benches/bench.rs +++ b/martin/benches/bench.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use criterion::async_executor::FuturesExecutor; use criterion::{criterion_group, criterion_main, Criterion}; -use martin::srv::get_tile_response; +use martin::srv::DynTileSource; use martin::{ CatalogSourceEntry, MartinResult, Source, TileCoord, TileData, TileSources, UrlQuery, }; @@ -58,7 +58,8 @@ impl Source for NullSource { } async fn process_tile(sources: &TileSources) { - get_tile_response(sources, TileCoord { z: 0, x: 0, y: 0 }, "null", "", None) + let src = DynTileSource::new(sources, "null", Some(0), "", None, None).unwrap(); + src.get_http_response(TileCoord { z: 0, x: 0, y: 0 }) .await .unwrap(); } diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 2fc83c2d5..7fc81d1be 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -43,6 +43,9 @@ pub struct MetaArgs { /// By default, only print if sources are auto-detected. #[arg(long)] pub save_config: Option, + /// Main cache size (in MB) + #[arg(short = 'C', long)] + pub cache_size: Option, /// **Deprecated** Scan for new sources on sources list requests #[arg(short, long, hide = true)] pub watch: bool, @@ -74,6 +77,10 @@ impl Args { return Err(ConfigAndConnectionsError(self.meta.connection)); } + if self.meta.cache_size.is_some() { + config.cache_size_mb = self.meta.cache_size; + } + self.srv.merge_into_config(&mut config.srv); #[allow(unused_mut)] diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 405d16bab..feecbfc95 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -12,10 +12,10 @@ use futures::stream::{self, StreamExt}; use futures::TryStreamExt; use log::{debug, error, info, log_enabled}; use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs}; -use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS}; +use martin::srv::{merge_tilejson, DynTileSource}; use martin::{ - append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source, - TileCoord, TileData, TileRect, + append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileCoord, + TileData, TileRect, }; use martin_tile_utils::{bbox_to_xyz, TileInfo}; use mbtiles::sqlx::SqliteConnection; @@ -144,7 +144,8 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; @@ -274,9 +275,18 @@ fn iterate_tiles(tiles: Vec) -> impl Iterator { async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> { let output_file = &args.output_file; let concurrency = args.concurrency.unwrap_or(1); - let (sources, _use_url_query, info) = state.tiles.get_sources(args.source.as_str(), None)?; - let sources = sources.as_slice(); - let tile_info = sources.first().unwrap().get_tile_info(); + + let src = DynTileSource::new( + &state.tiles, + args.source.as_str(), + None, + args.url_query.as_deref().unwrap_or_default(), + Some(parse_encoding(args.encoding.as_str())?), + None, + )?; + // parallel async below uses move, so we must only use copyable types + let src = &src; + let (tx, mut rx) = channel::(500); let tiles = compute_tile_ranges(&args); let mbt = Mbtiles::new(output_file)?; @@ -288,30 +298,26 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> } else { CopyDuplicateMode::Override }; - let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?; - let query = args.url_query.as_deref(); - let req = TestRequest::default() - .insert_header((ACCEPT_ENCODING, args.encoding.as_str())) - .finish(); - let accept_encoding = AcceptEncoding::parse(&req)?; - let encodings = Some(&accept_encoding); + let mbt_type = init_schema(&mbt, &mut conn, src.sources.as_slice(), src.info, &args).await?; let progress = Progress::new(&tiles); info!( - "Copying {} {tile_info} tiles from {} to {}", + "Copying {} {} tiles from {} to {}", progress.total, + src.info, args.source, args.output_file.display() ); try_join!( + // Note: for some reason, tests hang here without the `move` keyword async move { stream::iter(iterate_tiles(tiles)) .map(MartinResult::Ok) .try_for_each_concurrent(concurrency, |xyz| { let tx = tx.clone(); async move { - let tile = get_tile_content(sources, info, xyz, query, encodings).await?; + let tile = src.get_tile_content(xyz).await?; let data = tile.data; tx.send(TileXyz { xyz, data }) .await @@ -375,6 +381,13 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> Ok(()) } +fn parse_encoding(encoding: &str) -> MartinCpResult { + let req = TestRequest::default() + .insert_header((ACCEPT_ENCODING, encoding)) + .finish(); + Ok(AcceptEncoding::parse(&req)?) +} + async fn init_schema( mbt: &Mbtiles, conn: &mut SqliteConnection, diff --git a/martin/src/bin/martin.rs b/martin/src/bin/martin.rs index 5765e66e8..054970e49 100644 --- a/martin/src/bin/martin.rs +++ b/martin/src/bin/martin.rs @@ -4,8 +4,8 @@ use actix_web::dev::Server; use clap::Parser; use log::{error, info, log_enabled}; use martin::args::{Args, OsEnv}; -use martin::srv::{new_server, RESERVED_KEYWORDS}; -use martin::{read_config, Config, IdResolver, MartinResult}; +use martin::srv::new_server; +use martin::{read_config, Config, MartinResult}; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -24,7 +24,7 @@ async fn start(args: Args) -> MartinResult { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; diff --git a/martin/src/config.rs b/martin/src/config.rs index f311d5e69..8fed9347f 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -18,13 +18,15 @@ use crate::fonts::FontSources; use crate::source::{TileInfoSources, TileSources}; #[cfg(feature = "sprites")] use crate::sprites::{SpriteConfig, SpriteSources}; -use crate::srv::SrvConfig; +use crate::srv::{SrvConfig, RESERVED_KEYWORDS}; +use crate::utils::{CacheValue, MainCache, OptMainCache}; use crate::MartinError::{ConfigLoadError, ConfigParseError, ConfigWriteError, NoSources}; use crate::{IdResolver, MartinResult, OptOneMany}; pub type UnrecognizedValues = HashMap; pub struct ServerState { + pub cache: OptMainCache, pub tiles: TileSources, #[cfg(feature = "sprites")] pub sprites: SpriteSources, @@ -32,8 +34,11 @@ pub struct ServerState { pub fonts: FontSources, } +#[serde_with::skip_serializing_none] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Config { + pub cache_size_mb: Option, + #[serde(flatten)] pub srv: SrvConfig, @@ -107,19 +112,43 @@ impl Config { } } - pub async fn resolve(&mut self, idr: IdResolver) -> MartinResult { + pub async fn resolve(&mut self) -> MartinResult { + let resolver = IdResolver::new(RESERVED_KEYWORDS); + let cache_size = self.cache_size_mb.unwrap_or(512) * 1024 * 1024; + let cache = if cache_size > 0 { + info!("Initializing main cache with maximum size {cache_size}B"); + Some( + MainCache::builder() + .weigher(|_key, value: &CacheValue| -> u32 { + match value { + CacheValue::Tile(v) => v.len().try_into().unwrap_or(u32::MAX), + CacheValue::PmtDirectory(v) => { + v.get_approx_byte_size().try_into().unwrap_or(u32::MAX) + } + } + }) + .max_capacity(cache_size) + .build(), + ) + } else { + info!("Caching is disabled"); + None + }; + Ok(ServerState { - tiles: self.resolve_tile_sources(idr).await?, + tiles: self.resolve_tile_sources(&resolver, cache.clone()).await?, #[cfg(feature = "sprites")] sprites: SpriteSources::resolve(&mut self.sprites)?, #[cfg(feature = "fonts")] fonts: FontSources::resolve(&mut self.fonts)?, + cache, }) } async fn resolve_tile_sources( &mut self, - #[allow(unused_variables)] idr: IdResolver, + #[allow(unused_variables)] idr: &IdResolver, + #[allow(unused_variables)] cache: OptMainCache, ) -> MartinResult { #[allow(unused_mut)] let mut sources: Vec>>>> = @@ -133,14 +162,14 @@ impl Config { #[cfg(feature = "pmtiles")] if !self.pmtiles.is_empty() { let cfg = &mut self.pmtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "pmtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "pmtiles"); sources.push(Box::pin(val)); } #[cfg(feature = "mbtiles")] if !self.mbtiles.is_empty() { let cfg = &mut self.mbtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "mbtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "mbtiles"); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 636ec3f1c..3692f75be 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -14,7 +14,7 @@ use crate::file_config::FileError::{ InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError, }; use crate::source::{Source, TileInfoSources}; -use crate::utils::{IdResolver, OptOneMany}; +use crate::utils::{IdResolver, OptMainCache, OptOneMany}; use crate::MartinResult; use crate::OptOneMany::{Many, One}; @@ -48,7 +48,7 @@ pub enum FileError { } pub trait ConfigExtras: Clone + Debug + Default + PartialEq + Send { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, _cache: OptMainCache) -> FileResult<()> { Ok(()) } @@ -127,7 +127,10 @@ impl FileConfigEnum { } } - pub fn extract_file_config(&mut self) -> FileResult>> { + pub fn extract_file_config( + &mut self, + cache: OptMainCache, + ) -> FileResult>> { let mut res = match self { FileConfigEnum::None => return Ok(None), FileConfigEnum::Path(path) => FileConfig { @@ -140,7 +143,7 @@ impl FileConfigEnum { }, FileConfigEnum::Config(cfg) => mem::take(cfg), }; - res.custom.init_parsing()?; + res.custom.init_parsing(cache)?; Ok(Some(res)) } @@ -218,20 +221,22 @@ pub struct FileConfigSource { pub async fn resolve_files( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> MartinResult { - resolve_int(config, idr, extension) + resolve_int(config, idr, cache, extension) .map_err(crate::MartinError::from) .await } async fn resolve_int( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(cache)? else { return Ok(TileInfoSources::default()); }; diff --git a/martin/src/lib.rs b/martin/src/lib.rs index c0c9d4728..ef4aad9f7 100644 --- a/martin/src/lib.rs +++ b/martin/src/lib.rs @@ -10,7 +10,7 @@ pub use source::{CatalogSourceEntry, Source, Tile, TileData, TileSources, UrlQue mod utils; pub use utils::{ append_rect, decode_brotli, decode_gzip, IdResolver, MartinError, MartinResult, OptBoolObj, - OptOneMany, TileCoord, TileRect, + OptOneMany, TileCoord, TileRect, NO_MAIN_CACHE, }; pub mod args; diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index ec6514531..3a9ddf0d3 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -9,7 +9,6 @@ use std::sync::Arc; use async_trait::async_trait; use log::{trace, warn}; use martin_tile_utils::{Encoding, Format, TileInfo}; -use moka::future::Cache; use pmtiles::async_reader::AsyncPmTilesReader; use pmtiles::cache::{DirCacheResult, DirectoryCache}; use pmtiles::http::HttpBackend; @@ -24,20 +23,20 @@ use crate::config::UnrecognizedValues; use crate::file_config::FileError::{InvalidMetadata, InvalidUrlMetadata, IoError}; use crate::file_config::{ConfigExtras, FileError, FileResult, SourceConfigExtras}; use crate::source::UrlQuery; +use crate::utils::cache::get_cached_value; +use crate::utils::{CacheKey, CacheValue, OptMainCache}; use crate::{MartinResult, Source, TileCoord, TileData}; -type PmtCacheObject = Cache<(usize, usize), Directory>; - #[derive(Clone, Debug)] pub struct PmtCache { id: usize, - /// (id, offset) -> Directory, or None to disable caching - cache: Option, + /// Storing (id, offset) -> Directory, or None to disable caching + cache: OptMainCache, } impl PmtCache { #[must_use] - pub fn new(id: usize, cache: Option) -> Self { + pub fn new(id: usize, cache: OptMainCache) -> Self { Self { id, cache } } } @@ -45,17 +44,23 @@ impl PmtCache { #[async_trait] impl DirectoryCache for PmtCache { async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { - if let Some(cache) = &self.cache { - if let Some(dir) = cache.get(&(self.id, offset)).await { - return dir.find_tile_id(tile_id).into(); - } + if let Some(dir) = get_cached_value!(&self.cache, CacheValue::PmtDirectory, { + CacheKey::PmtDirectory(self.id, offset) + }) { + dir.find_tile_id(tile_id).into() + } else { + DirCacheResult::NotCached } - DirCacheResult::NotCached } async fn insert_dir(&self, offset: usize, directory: Directory) { if let Some(cache) = &self.cache { - cache.insert((self.id, offset), directory).await; + cache + .insert( + CacheKey::PmtDirectory(self.id, offset), + CacheValue::PmtDirectory(directory), + ) + .await; } } } @@ -63,8 +68,6 @@ impl DirectoryCache for PmtCache { #[serde_with::skip_serializing_none] #[derive(Debug, Default, Serialize, Deserialize)] pub struct PmtConfig { - pub dir_cache_size_mb: Option, - #[serde(flatten)] pub unrecognized: UnrecognizedValues, @@ -78,12 +81,12 @@ pub struct PmtConfig { pub next_cache_id: AtomicUsize, #[serde(skip)] - pub cache: Option, + pub cache: OptMainCache, } impl PartialEq for PmtConfig { fn eq(&self, other: &Self) -> bool { - self.dir_cache_size_mb == other.dir_cache_size_mb && self.unrecognized == other.unrecognized + self.unrecognized == other.unrecognized } } @@ -91,7 +94,6 @@ impl Clone for PmtConfig { fn clone(&self) -> Self { // State is not shared between clones, only the serialized config Self { - dir_cache_size_mb: self.dir_cache_size_mb, unrecognized: self.unrecognized.clone(), ..Default::default() } @@ -107,24 +109,17 @@ impl PmtConfig { } impl ConfigExtras for PmtConfig { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, cache: OptMainCache) -> FileResult<()> { assert!(self.client.is_none()); assert!(self.cache.is_none()); self.client = Some(Client::new()); + self.cache = cache; - // Allow cache size to be disabled with 0 - let dir_cache_size = self.dir_cache_size_mb.unwrap_or(32) * 1024 * 1024; - if dir_cache_size > 0 { - self.cache = Some( - Cache::builder() - .weigher(|_key, value: &Directory| -> u32 { - value.get_approx_byte_size().try_into().unwrap_or(u32::MAX) - }) - .max_capacity(dir_cache_size) - .build(), - ); + if self.unrecognized.contains_key("dir_cache_size_mb") { + warn!("dir_cache_size_mb is no longer used. Instead, use cache_size_mb param in the root of the config file."); } + Ok(()) } diff --git a/martin/src/source.rs b/martin/src/source.rs index 01c2bc2b5..ba04293b9 100644 --- a/martin/src/source.rs +++ b/martin/src/source.rs @@ -167,6 +167,7 @@ mod tests { } } +#[derive(Debug, Clone)] pub struct Tile { pub data: TileData, pub info: TileInfo, diff --git a/martin/src/sprites/mod.rs b/martin/src/sprites/mod.rs index fa81f179b..7a0fc8cc4 100644 --- a/martin/src/sprites/mod.rs +++ b/martin/src/sprites/mod.rs @@ -77,7 +77,7 @@ pub struct SpriteSources(HashMap); impl SpriteSources { pub fn resolve(config: &mut FileConfigEnum) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(None)? else { return Ok(Self::default()); }; diff --git a/martin/src/srv/fonts.rs b/martin/src/srv/fonts.rs old mode 100755 new mode 100644 diff --git a/martin/src/srv/mod.rs b/martin/src/srv/mod.rs index ea94de2ea..b9ad026f8 100644 --- a/martin/src/srv/mod.rs +++ b/martin/src/srv/mod.rs @@ -1,14 +1,14 @@ mod config; pub use config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT}; +#[cfg(feature = "fonts")] +mod fonts; + mod server; pub use server::{new_server, router, Catalog, RESERVED_KEYWORDS}; mod tiles; -pub use tiles::{get_tile_content, get_tile_response, TileRequest}; - -#[cfg(feature = "fonts")] -mod fonts; +pub use tiles::{DynTileSource, TileRequest}; mod tiles_info; pub use tiles_info::{merge_tilejson, SourceIDsRequest}; diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index 377638b66..4fe0e4d9d 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -112,7 +112,9 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server .allow_any_origin() .allowed_methods(vec!["GET"]); - let app = App::new().app_data(Data::new(state.tiles.clone())); + let app = App::new() + .app_data(Data::new(state.tiles.clone())) + .app_data(Data::new(state.cache.clone())); #[cfg(feature = "sprites")] let app = app.app_data(Data::new(state.sprites.clone())); diff --git a/martin/src/srv/sprites.rs b/martin/src/srv/sprites.rs index c3275bbeb..55a61a81b 100644 --- a/martin/src/srv/sprites.rs +++ b/martin/src/srv/sprites.rs @@ -4,28 +4,18 @@ use actix_web::error::ErrorNotFound; use actix_web::http::header::ContentType; use actix_web::web::{Data, Path}; use actix_web::{middleware, route, HttpResponse, Result as ActixResult}; +use spreet::Spritesheet; use crate::sprites::{SpriteError, SpriteSources}; use crate::srv::server::map_internal_error; use crate::srv::SourceIDsRequest; -pub fn map_sprite_error(e: SpriteError) -> actix_web::Error { - use SpriteError::SpriteNotFound; - match e { - SpriteNotFound(_) => ErrorNotFound(e.to_string()), - _ => map_internal_error(e), - } -} - #[route("/sprite/{source_ids}.png", method = "GET", method = "HEAD")] async fn get_sprite_png( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites - .get_sprites(&path.source_ids) - .await - .map_err(map_sprite_error)?; + let sheet = get_sprite(&path, &sprites).await?; Ok(HttpResponse::Ok() .content_type(ContentType::png()) .body(sheet.encode_png().map_err(map_internal_error)?)) @@ -41,9 +31,16 @@ async fn get_sprite_json( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites + let sheet = get_sprite(&path, &sprites).await?; + Ok(HttpResponse::Ok().json(sheet.get_index())) +} + +async fn get_sprite(path: &SourceIDsRequest, sprites: &SpriteSources) -> ActixResult { + sprites .get_sprites(&path.source_ids) .await - .map_err(map_sprite_error)?; - Ok(HttpResponse::Ok().json(sheet.get_index())) + .map_err(|e| match e { + SpriteError::SpriteNotFound(_) => ErrorNotFound(e.to_string()), + _ => map_internal_error(e), + }) } diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs old mode 100644 new mode 100755 index 32463a6f3..f25829966 --- a/martin/src/srv/tiles.rs +++ b/martin/src/srv/tiles.rs @@ -6,13 +6,18 @@ use actix_web::http::header::{ use actix_web::web::{Data, Path, Query}; use actix_web::{route, HttpMessage, HttpRequest, HttpResponse, Result as ActixResult}; use futures::future::try_join_all; +use log::trace; use martin_tile_utils::{Encoding, Format, TileInfo}; use serde::Deserialize; use crate::source::{Source, TileSources, UrlQuery}; use crate::srv::server::map_internal_error; -use crate::utils::{decode_brotli, decode_gzip, encode_brotli, encode_gzip}; -use crate::{Tile, TileCoord}; +use crate::utils::cache::get_or_insert_cached_value; +use crate::utils::{ + decode_brotli, decode_gzip, encode_brotli, encode_gzip, CacheKey, CacheValue, MainCache, + OptMainCache, +}; +use crate::{Tile, TileCoord, TileData}; static SUPPORTED_ENCODINGS: &[HeaderEnc] = &[ HeaderEnc::brotli(), @@ -33,125 +38,165 @@ async fn get_tile( req: HttpRequest, path: Path, sources: Data, + cache: Data, ) -> ActixResult { - let xyz = TileCoord { + let src = DynTileSource::new( + sources.as_ref(), + &path.source_ids, + Some(path.z), + req.query_string(), + req.get_header::(), + cache.as_ref().as_ref(), + )?; + + src.get_http_response(TileCoord { z: path.z, x: path.x, y: path.y, - }; - - let source_ids = &path.source_ids; - let query = req.query_string(); - let encodings = req.get_header::(); + }) + .await +} - get_tile_response(sources.as_ref(), xyz, source_ids, query, encodings).await +pub struct DynTileSource<'a> { + pub sources: Vec<&'a dyn Source>, + pub info: TileInfo, + pub query_str: Option<&'a str>, + pub query_obj: Option, + pub encodings: Option, + pub cache: Option<&'a MainCache>, } -pub async fn get_tile_response( - sources: &TileSources, - xyz: TileCoord, - source_ids: &str, - query: &str, - encodings: Option, -) -> ActixResult { - let (sources, use_url_query, info) = sources.get_sources(source_ids, Some(xyz.z))?; +impl<'a> DynTileSource<'a> { + pub fn new( + sources: &'a TileSources, + source_ids: &str, + zoom: Option, + query: &'a str, + encodings: Option, + cache: Option<&'a MainCache>, + ) -> ActixResult { + let (sources, use_url_query, info) = sources.get_sources(source_ids, zoom)?; - let query = use_url_query.then_some(query); - let tile = get_tile_content(sources.as_slice(), info, xyz, query, encodings.as_ref()).await?; + if sources.is_empty() { + return Err(ErrorNotFound("No valid sources found")); + } - Ok(if tile.data.is_empty() { - HttpResponse::NoContent().finish() - } else { - let mut response = HttpResponse::Ok(); - response.content_type(tile.info.format.content_type()); - if let Some(val) = tile.info.encoding.content_encoding() { - response.insert_header((CONTENT_ENCODING, val)); + let mut query_obj = None; + let mut query_str = None; + if use_url_query && !query.is_empty() { + query_obj = Some(Query::::from_query(query)?.into_inner()); + query_str = Some(query); } - response.body(tile.data) - }) -} -pub async fn get_tile_content( - sources: &[&dyn Source], - info: TileInfo, - xyz: TileCoord, - query: Option<&str>, - encodings: Option<&AcceptEncoding>, -) -> ActixResult { - if sources.is_empty() { - return Err(ErrorNotFound("No valid sources found")); + Ok(Self { + sources, + info, + query_str, + query_obj, + encodings, + cache, + }) } - let query_str = query.filter(|v| !v.is_empty()); - let query = match query_str { - Some(v) => Some(Query::::from_query(v)?.into_inner()), - None => None, - }; - let mut tiles = try_join_all(sources.iter().map(|s| s.get_tile(xyz, query.as_ref()))) - .await - .map_err(map_internal_error)?; + pub async fn get_http_response(&self, xyz: TileCoord) -> ActixResult { + let tile = self.get_tile_content(xyz).await?; - let mut layer_count = 0; - let mut last_non_empty_layer = 0; - for (idx, tile) in tiles.iter().enumerate() { - if !tile.is_empty() { - layer_count += 1; - last_non_empty_layer = idx; - } + Ok(if tile.data.is_empty() { + HttpResponse::NoContent().finish() + } else { + let mut response = HttpResponse::Ok(); + response.content_type(tile.info.format.content_type()); + if let Some(val) = tile.info.encoding.content_encoding() { + response.insert_header((CONTENT_ENCODING, val)); + } + response.body(tile.data) + }) } - // Minor optimization to prevent concatenation if there are less than 2 tiles - let data = match layer_count { - 1 => tiles.swap_remove(last_non_empty_layer), - 0 => return Ok(Tile::new(Vec::new(), info)), - _ => { - // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level - // TODO: can zlib, brotli, or zstd be concatenated? - // TODO: implement decompression step for other concatenate-able formats - let can_join = info.format == Format::Mvt - && (info.encoding == Encoding::Uncompressed || info.encoding == Encoding::Gzip); - if !can_join { - return Err(ErrorBadRequest(format!( - "Can't merge {info} tiles. Make sure there is only one non-empty tile source at zoom level {}", - xyz.z - )))?; + pub async fn get_tile_content(&self, xyz: TileCoord) -> ActixResult { + let mut tiles = try_join_all(self.sources.iter().map(|s| async { + get_or_insert_cached_value!( + self.cache, + CacheValue::Tile, + s.get_tile(xyz, self.query_obj.as_ref()), + { + let id = s.get_id().to_owned(); + if let Some(query_str) = self.query_str { + CacheKey::TileWithQuery(id, xyz, query_str.to_owned()) + } else { + CacheKey::Tile(id, xyz) + } + } + ) + })) + .await + .map_err(map_internal_error)?; + + let mut layer_count = 0; + let mut last_non_empty_layer = 0; + for (idx, tile) in tiles.iter().enumerate() { + if !tile.is_empty() { + layer_count += 1; + last_non_empty_layer = idx; } - tiles.concat() } - }; - // decide if (re-)encoding of the tile data is needed, and recompress if so - let tile = recompress(Tile::new(data, info), encodings)?; + // Minor optimization to prevent concatenation if there are less than 2 tiles + let data = match layer_count { + 1 => tiles.swap_remove(last_non_empty_layer), + 0 => return Ok(Tile::new(Vec::new(), self.info)), + _ => { + // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level + // TODO: can zlib, brotli, or zstd be concatenated? + // TODO: implement decompression step for other concatenate-able formats + let can_join = self.info.format == Format::Mvt + && (self.info.encoding == Encoding::Uncompressed + || self.info.encoding == Encoding::Gzip); + if !can_join { + return Err(ErrorBadRequest(format!( + "Can't merge {} tiles. Make sure there is only one non-empty tile source at zoom level {}", + self.info, + xyz.z + )))?; + } + tiles.concat() + } + }; - Ok(tile) -} + // decide if (re-)encoding of the tile data is needed, and recompress if so + self.recompress(data) + } -fn recompress(mut tile: Tile, accept_enc: Option<&AcceptEncoding>) -> ActixResult { - if let Some(accept_enc) = accept_enc { - if tile.info.encoding.is_encoded() { - // already compressed, see if we can send it as is, or need to re-compress - if !accept_enc.iter().any(|e| { - if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { - to_encoding(enc) == Some(tile.info.encoding) - } else { - false + fn recompress(&self, tile: TileData) -> ActixResult { + let mut tile = Tile::new(tile, self.info); + if let Some(accept_enc) = &self.encodings { + if self.info.encoding.is_encoded() { + // already compressed, see if we can send it as is, or need to re-compress + if !accept_enc.iter().any(|e| { + if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { + to_encoding(enc) == Some(tile.info.encoding) + } else { + false + } + }) { + // need to re-compress the tile - uncompress it first + tile = decode(tile)?; } - }) { - // need to re-compress the tile - uncompress it first - tile = decode(tile)?; } - } - if tile.info.encoding == Encoding::Uncompressed { - // only apply compression if the content supports it - if let Some(HeaderEnc::Known(enc)) = accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) { - // (re-)compress the tile into the preferred encoding - tile = encode(tile, enc)?; + if tile.info.encoding == Encoding::Uncompressed { + // only apply compression if the content supports it + if let Some(HeaderEnc::Known(enc)) = + accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) + { + // (re-)compress the tile into the preferred encoding + tile = encode(tile, enc)?; + } } + Ok(tile) + } else { + // no accepted-encoding header, decode the tile if compressed + decode(tile) } - Ok(tile) - } else { - // no accepted-encoding header, decode the tile if compressed - decode(tile) } } @@ -189,7 +234,7 @@ fn decode(tile: Tile) -> ActixResult { }) } -fn to_encoding(val: ContentEncoding) -> Option { +pub fn to_encoding(val: ContentEncoding) -> Option { Some(match val { ContentEncoding::Identity => Encoding::Uncompressed, ContentEncoding::Gzip => Encoding::Gzip, @@ -233,15 +278,9 @@ mod tests { ("empty,non-empty", vec![1_u8, 2, 3]), ("empty,non-empty,empty", vec![1_u8, 2, 3]), ] { - let (src, _, info) = sources.get_sources(source_id, None).unwrap(); + let src = DynTileSource::new(&sources, source_id, None, "", None, None).unwrap(); let xyz = TileCoord { z: 0, x: 0, y: 0 }; - assert_eq!( - expected, - &get_tile_content(src.as_slice(), info, xyz, None, None) - .await - .unwrap() - .data - ); + assert_eq!(expected, &src.get_tile_content(xyz).await.unwrap().data); } } } diff --git a/martin/src/srv/tiles_info.rs b/martin/src/srv/tiles_info.rs old mode 100644 new mode 100755 diff --git a/martin/src/utils/cache.rs b/martin/src/utils/cache.rs new file mode 100755 index 000000000..ff4421569 --- /dev/null +++ b/martin/src/utils/cache.rs @@ -0,0 +1,91 @@ +use moka::future::Cache; +use pmtiles::Directory; + +use crate::{TileCoord, TileData}; + +pub type MainCache = Cache; +pub type OptMainCache = Option; +pub const NO_MAIN_CACHE: OptMainCache = None; + +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum CacheKey { + /// (pmtiles_id, offset) + PmtDirectory(usize, usize), + /// (source_id, xyz) + Tile(String, TileCoord), + /// (source_id, xyz, url_query) + TileWithQuery(String, TileCoord, String), +} + +#[derive(Debug, Clone)] +pub enum CacheValue { + Tile(TileData), + PmtDirectory(Directory), +} + +macro_rules! trace_cache { + ($typ: literal, $cache: expr, $key: expr) => { + trace!( + "Cache {} for {:?} in {:?} that has {} entries taking up {} space", + $typ, + $key, + $cache.name(), + $cache.entry_count(), + $cache.weighted_size(), + ); + }; +} + +macro_rules! from_cache_value { + ($value_type: path, $data: expr, $key: expr) => { + if let $value_type(data) = $data { + data + } else { + panic!("Unexpected value type {:?} for key {:?} cache", $data, $key) + } + }; +} +#[cfg(feature = "pmtiles")] +macro_rules! get_cached_value { + ($cache: expr, $value_type: path, $make_key: expr) => { + if let Some(cache) = $cache { + let key = $make_key; + if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + Some($crate::utils::cache::from_cache_value!( + $value_type, + data, + key + )) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + None + } + } else { + None + } + }; +} + +macro_rules! get_or_insert_cached_value { + ($cache: expr, $value_type: path, $make_item:expr, $make_key: expr) => {{ + if let Some(cache) = $cache { + let key = $make_key; + Ok(if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + $crate::utils::cache::from_cache_value!($value_type, data, key) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + let data = $make_item.await?; + cache.insert(key, $value_type(data.clone())).await; + data + }) + } else { + $make_item.await + } + }}; +} + +#[cfg(feature = "pmtiles")] +pub(crate) use get_cached_value; +pub(crate) use {from_cache_value, get_or_insert_cached_value, trace_cache}; diff --git a/martin/src/utils/mod.rs b/martin/src/utils/mod.rs index e0444cdee..306da7e55 100644 --- a/martin/src/utils/mod.rs +++ b/martin/src/utils/mod.rs @@ -1,3 +1,6 @@ +pub(crate) mod cache; +pub use cache::{CacheKey, CacheValue, MainCache, OptMainCache, NO_MAIN_CACHE}; + mod cfg_containers; pub use cfg_containers::{OptBoolObj, OptOneMany}; diff --git a/martin/src/utils/xyz.rs b/martin/src/utils/xyz.rs index 421ec6df6..1a968207e 100644 --- a/martin/src/utils/xyz.rs +++ b/martin/src/utils/xyz.rs @@ -1,6 +1,6 @@ use std::fmt::{Display, Formatter}; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct TileCoord { pub z: u8, pub x: u32, diff --git a/martin/tests/mb_server_test.rs b/martin/tests/mb_server_test.rs index b20b64f5b..b38d490b6 100644 --- a/martin/tests/mb_server_test.rs +++ b/martin/tests/mb_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index ebbc5114a..55b7f37a3 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -26,6 +26,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) @@ -1086,6 +1087,7 @@ tables: .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pmt_server_test.rs b/martin/tests/pmt_server_test.rs index 5b1a4d197..9a6e7ce46 100644 --- a/martin/tests/pmt_server_test.rs +++ b/martin/tests/pmt_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs index 334b1179f..02116664b 100644 --- a/martin/tests/utils/pg_utils.rs +++ b/martin/tests/utils/pg_utils.rs @@ -1,6 +1,6 @@ use indoc::formatdoc; pub use martin::args::Env; -use martin::{Config, IdResolver, ServerState, Source}; +use martin::{Config, ServerState, Source}; use crate::mock_cfg; @@ -22,7 +22,7 @@ pub fn mock_pgcfg(yaml: &str) -> Config { #[allow(dead_code)] pub async fn mock_sources(mut config: Config) -> MockSource { - let res = config.resolve(IdResolver::default()).await; + let res = config.resolve().await; let res = res.unwrap_or_else(|e| panic!("Failed to resolve config {config:?}: {e}")); (res, config) } diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index f30213609..1f22df562 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/tests/config.yaml b/tests/config.yaml index 7a3f28488..70b08a05c 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -8,6 +8,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 8 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string @@ -166,7 +169,6 @@ postgres: pmtiles: - dir_cache_size_mb: 100 paths: - http://localhost:5412/webp2.pmtiles sources: diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 47379599a..308490c10 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -1,3 +1,4 @@ +cache_size_mb: 8 keep_alive: 75 listen_addresses: localhost:3111 worker_processes: 1 @@ -165,7 +166,6 @@ pmtiles: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles pmt2: http://localhost:5412/webp2.pmtiles webp2: http://localhost:5412/webp2.pmtiles - dir_cache_size_mb: 100 sprites: paths: tests/fixtures/sprites/src1 sources: