diff --git a/Cargo.lock b/Cargo.lock index 75ed2c211..ca403046d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1693,7 +1693,7 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "martin" -version = "0.9.3" +version = "0.10.0" dependencies = [ "actix-cors", "actix-http", diff --git a/debian/config.yaml b/debian/config.yaml index aacb3b20a..2f7f99cf1 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -14,7 +14,7 @@ worker_processes: 8 # default_srid: 4326 # pool_size: 20 # max_feature_count: 1000 -# disable_bounds: false +# auto_bounds: skip # pmtiles: # paths: diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 8fe58a638..deaeba64d 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -47,9 +47,11 @@ postgres: # Limit the number of table geo features included in a tile. Unlimited by default. max_feature_count: 1000 - # Control the automatic generation of bounds for spatial tables [default: false] - # If enabled, it will spend some time on startup to compute geometry bounds. - disable_bounds: false + # Control the automatic generation of bounds for spatial tables [default: quick] + # 'calc' - compute table geometry bounds on startup. + # 'quick' - same as 'calc', but the calculation will be aborted if it takes more than 5 seconds. + # 'skip' - do not compute table geometry bounds on startup. + auto_bounds: skip # Enable automatic discovery of tables and functions. # You may set this to `false` to disable. diff --git a/docs/src/run-with-cli.md b/docs/src/run-with-cli.md index d8c235654..c51d2a139 100644 --- a/docs/src/run-with-cli.md +++ b/docs/src/run-with-cli.md @@ -6,33 +6,51 @@ You can configure Martin using command-line interface. See `martin --help` or `c Usage: martin [OPTIONS] [CONNECTION]... Arguments: - [CONNECTION]... Connection strings, e.g. postgres://... or /path/to/files + [CONNECTION]... + Connection strings, e.g. postgres://... or /path/to/files Options: -c, --config Path to config file. If set, no tile source-related parameters are allowed + --save-config Save resulting config to a file or use "-" to print to stdout. By default, only print if sources are auto-detected + -s, --sprite Export a directory with SVG files as a sprite source. Can be specified multiple times + -k, --keep-alive Connection keep alive timeout. [DEFAULT: 75] + -l, --listen-addresses The socket address to bind. [DEFAULT: 0.0.0.0:3000] + -W, --workers Number of web server workers - -b, --disable-bounds - Disable the automatic generation of bounds for spatial PG tables + + -b, --auto-bounds + Specify how bounds should be computed for the spatial PG tables. [DEFAULT: quick] + + Possible values: + - quick: Compute table geometry bounds, but abort if it takes longer than 5 seconds + - calc: Compute table geometry bounds. The startup time may be significant. Make sure all GEO columns have indexes + - skip: Skip bounds calculation. The bounds will be set to the whole world + --ca-root-file Loads trusted root certificates from a file. The file should contain a sequence of PEM-formatted CA certificates + -d, --default-srid If a spatial PG table has SRID 0, then this default SRID will be used as a fallback + -p, --pool-size Maximum connections pool size [DEFAULT: 20] + -m, --max-feature-count Limit the number of features in a tile from a PG table source + -h, --help - Print help + Print help (see a summary with '-h') + -V, --version Print version ``` diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 38681e00b..800b24139 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "martin" # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin -version = "0.9.3" +version = "0.10.0" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"] diff --git a/martin/src/args/mod.rs b/martin/src/args/mod.rs index 6866d87ba..daa8a6567 100644 --- a/martin/src/args/mod.rs +++ b/martin/src/args/mod.rs @@ -6,4 +6,5 @@ mod srv; pub use connections::{Arguments, State}; pub use environment::{Env, OsEnv}; -pub use root::Args; +pub use pg::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT}; +pub use root::{Args, MetaArgs}; diff --git a/martin/src/args/pg.rs b/martin/src/args/pg.rs index 0ca12136e..2cf18c711 100644 --- a/martin/src/args/pg.rs +++ b/martin/src/args/pg.rs @@ -1,4 +1,8 @@ +use std::time::Duration; + +use clap::ValueEnum; use log::{info, warn}; +use serde::{Deserialize, Serialize}; use crate::args::connections::Arguments; use crate::args::connections::State::{Ignore, Take}; @@ -6,12 +10,27 @@ use crate::args::environment::Env; use crate::pg::{PgConfig, PgSslCerts, POOL_SIZE_DEFAULT}; use crate::utils::{OptBoolObj, OptOneMany}; +// Must match the help string for BoundsType::Quick +pub const DEFAULT_BOUNDS_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(PartialEq, Eq, Default, Debug, Clone, Copy, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum BoundsCalcType { + /// Compute table geometry bounds, but abort if it takes longer than 5 seconds. + #[default] + Quick, + /// Compute table geometry bounds. The startup time may be significant. Make sure all GEO columns have indexes. + Calc, + /// Skip bounds calculation. The bounds will be set to the whole world. + Skip, +} + #[derive(clap::Args, Debug, PartialEq, Default)] #[command(about, version)] pub struct PgArgs { - /// Disable the automatic generation of bounds for spatial PG tables. + /// Specify how bounds should be computed for the spatial PG tables. [DEFAULT: quick] #[arg(short = 'b', long)] - pub disable_bounds: bool, + pub auto_bounds: Option, /// Loads trusted root certificates from a file. The file should contain a sequence of PEM-formatted CA certificates. #[arg(long)] pub ca_root_file: Option, @@ -41,11 +60,7 @@ impl PgArgs { connection_string: Some(s), ssl_certificates: certs.clone(), default_srid, - disable_bounds: if self.disable_bounds { - Some(true) - } else { - None - }, + auto_bounds: self.auto_bounds, max_feature_count: self.max_feature_count, pool_size: self.pool_size, auto_publish: OptBoolObj::NoValue, diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 844e213c8..98c82560b 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -36,7 +36,7 @@ pub struct MetaArgs { /// By default, only print if sources are auto-detected. #[arg(long)] pub save_config: Option, - /// [Deprecated] Scan for new sources on sources list requests + /// **Deprecated** Scan for new sources on sources list requests #[arg(short, long, hide = true)] pub watch: bool, /// Connection strings, e.g. postgres://... or /path/to/files diff --git a/martin/src/pg/config.rs b/martin/src/pg/config.rs index 51de0a394..5dfa3c1e8 100644 --- a/martin/src/pg/config.rs +++ b/martin/src/pg/config.rs @@ -1,3 +1,4 @@ +use std::ops::Add; use std::time::Duration; use futures::future::try_join; @@ -5,6 +6,7 @@ use log::warn; use serde::{Deserialize, Serialize}; use tilejson::TileJSON; +use crate::args::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT}; use crate::config::{copy_unrecognized_config, UnrecognizedValues}; use crate::pg::config_function::FuncInfoSources; use crate::pg::config_table::TableInfoSources; @@ -42,7 +44,7 @@ pub struct PgConfig { #[serde(skip_serializing_if = "Option::is_none")] pub default_srid: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub disable_bounds: Option, + pub auto_bounds: Option, #[serde(skip_serializing_if = "Option::is_none")] pub max_feature_count: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -123,13 +125,18 @@ impl PgConfig { pub async fn resolve(&mut self, id_resolver: IdResolver) -> crate::Result { let pg = PgBuilder::new(self, id_resolver).await?; - let inst_tables = on_slow(pg.instantiate_tables(), Duration::from_secs(5), || { - if pg.disable_bounds() { - warn!("Discovering tables in PostgreSQL database '{}' is taking too long. Bounds calculation is already disabled. You may need to tune your database.", pg.get_id()); - } else { - warn!("Discovering tables in PostgreSQL database '{}' is taking too long. Make sure your table geo columns have a GIS index, or use --disable-bounds CLI/config to skip bbox calculation.", pg.get_id()); - } - }); + let inst_tables = on_slow( + pg.instantiate_tables(), + // warn only if default bounds timeout has already passed + DEFAULT_BOUNDS_TIMEOUT.add(Duration::from_secs(1)), + || { + if pg.auto_bounds() == BoundsCalcType::Skip { + warn!("Discovering tables in PostgreSQL database '{}' is taking too long. Make sure your table geo columns have a GIS index, or use '--auto-bounds skip' CLI/config to skip bbox calculation.", pg.get_id()); + } else { + warn!("Discovering tables in PostgreSQL database '{}' is taking too long. Bounds calculation is already disabled. You may need to tune your database.", pg.get_id()); + } + }, + ); let ((mut tables, tbl_info), (funcs, func_info)) = try_join(inst_tables, pg.instantiate_functions()).await?; diff --git a/martin/src/pg/configurator.rs b/martin/src/pg/configurator.rs index 7a67a1fc2..cefbdba40 100644 --- a/martin/src/pg/configurator.rs +++ b/martin/src/pg/configurator.rs @@ -5,6 +5,7 @@ use futures::future::join_all; use itertools::Itertools; use log::{debug, error, info, warn}; +use crate::args::BoundsCalcType; use crate::pg::config::{PgConfig, PgInfo}; use crate::pg::config_function::{FuncInfoSources, FunctionInfo}; use crate::pg::config_table::{TableInfo, TableInfoSources}; @@ -59,7 +60,7 @@ pub struct PgBuilderTables { pub struct PgBuilder { pool: PgPool, default_srid: Option, - disable_bounds: bool, + auto_bounds: BoundsCalcType, max_feature_count: Option, auto_functions: Option, auto_tables: Option, @@ -97,7 +98,7 @@ impl PgBuilder { Ok(Self { pool, default_srid: config.default_srid, - disable_bounds: config.disable_bounds.unwrap_or_default(), + auto_bounds: config.auto_bounds.unwrap_or_default(), max_feature_count: config.max_feature_count, id_resolver, tables: config.tables.clone().unwrap_or_default(), @@ -107,8 +108,8 @@ impl PgBuilder { }) } - pub fn disable_bounds(&self) -> bool { - self.disable_bounds + pub fn auto_bounds(&self) -> BoundsCalcType { + self.auto_bounds } pub fn get_id(&self) -> &str { @@ -160,7 +161,7 @@ impl PgBuilder { id2, merged_inf, self.pool.clone(), - self.disable_bounds, + self.auto_bounds, self.max_feature_count, )); } @@ -206,7 +207,7 @@ impl PgBuilder { id2, db_inf, self.pool.clone(), - self.disable_bounds, + self.auto_bounds, self.max_feature_count, )); } diff --git a/martin/src/pg/table_source.rs b/martin/src/pg/table_source.rs index d26adad85..5b95b58f7 100644 --- a/martin/src/pg/table_source.rs +++ b/martin/src/pg/table_source.rs @@ -1,11 +1,14 @@ use std::collections::HashMap; +use futures::pin_mut; use log::{debug, info, warn}; use postgis::ewkb; use postgres_protocol::escape::{escape_identifier, escape_literal}; use serde_json::Value; use tilejson::Bounds; +use tokio::time::timeout; +use crate::args::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT}; use crate::pg::config::PgInfo; use crate::pg::config_table::TableInfo; use crate::pg::configurator::SqlTableInfoMapMapMap; @@ -96,7 +99,7 @@ pub async fn table_to_query( id: String, mut info: TableInfo, pool: PgPool, - disable_bounds: bool, + bounds_type: BoundsCalcType, max_feature_count: Option, ) -> Result<(String, PgSqlInfo, TableInfo)> { let schema = escape_identifier(&info.schema); @@ -104,8 +107,26 @@ pub async fn table_to_query( let geometry_column = escape_identifier(&info.geometry_column); let srid = info.srid; - if info.bounds.is_none() && !disable_bounds { - info.bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid).await?; + if info.bounds.is_none() { + match bounds_type { + BoundsCalcType::Skip => {} + BoundsCalcType::Quick | BoundsCalcType::Calc => { + let bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid); + if bounds_type == BoundsCalcType::Calc { + info.bounds = bounds.await?; + } else { + pin_mut!(bounds); + if let Ok(bounds) = timeout(DEFAULT_BOUNDS_TIMEOUT, &mut bounds).await { + info.bounds = bounds?; + } else { + warn!( + "Timeout computing {} bounds for {id}, aborting query. Use --auto-bounds=calc to wait until complete, or check the table for missing indices.", + info.format_id(), + ); + } + } + } + } } let properties = if let Some(props) = &info.properties { diff --git a/tests/expected/auto/cmp.json b/tests/expected/auto/cmp.json index 234b2bc3d..161878bc1 100644 --- a/tests/expected/auto/cmp.json +++ b/tests/expected/auto/cmp.json @@ -23,6 +23,12 @@ } } ], + "bounds": [ + -179.27313970132585, + -80.46177157848345, + 179.11187181086706, + 84.93092095128937 + ], "description": "public.points1.geom\npublic.points2.geom", "name": "table_source,points1,points2" } diff --git a/tests/expected/auto/points3857_srid.json b/tests/expected/auto/points3857_srid.json index 195123bc5..4b90db678 100644 --- a/tests/expected/auto/points3857_srid.json +++ b/tests/expected/auto/points3857_srid.json @@ -11,6 +11,12 @@ } } ], + "bounds": [ + -161.40590777554058, + -81.50727021609012, + 172.51549126768532, + 84.2440187164111 + ], "description": "public.points3857.geom", "name": "points3857" } diff --git a/tests/expected/auto/table_source.json b/tests/expected/auto/table_source.json index 588e57b37..609c974b5 100644 --- a/tests/expected/auto/table_source.json +++ b/tests/expected/auto/table_source.json @@ -11,6 +11,12 @@ } } ], + "bounds": [ + -2, + -1, + 142.84131509869133, + 45 + ], "name": "table_source", "foo": { "bar": "foo" diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index e8229fd9a..50bf43f96 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -1,7 +1,7 @@ listen_addresses: localhost:3111 postgres: default_srid: 900913 - disable_bounds: true + auto_bounds: calc auto_publish: true tables: MixPoints: @@ -9,6 +9,11 @@ postgres: table: MixPoints srid: 4326 geometry_column: Geom + bounds: + - -170.94984639004662 + - -84.20025580733805 + - 167.70892858284475 + - 74.23573284753762 geometry_type: POINT properties: Gid: int4 @@ -18,6 +23,11 @@ postgres: table: auto_table srid: 4326 geometry_column: geom + bounds: + - -166.87107126230424 + - -53.44747249115674 + - 168.14061220360549 + - 84.22411861475385 geometry_type: POINT properties: feat_id: int4 @@ -27,6 +37,11 @@ postgres: table: bigint_table srid: 4326 geometry_column: geom + bounds: + - -174.89475564568033 + - -77.2579745396886 + - 174.72753224514435 + - 73.80785950599903 geometry_type: POINT properties: big_feat_id: int8 @@ -36,6 +51,11 @@ postgres: table: points1 srid: 4326 geometry_column: geom + bounds: + - -179.27313970132585 + - -67.52518563265659 + - 162.60117193735186 + - 84.93092095128937 geometry_type: POINT properties: gid: int4 @@ -44,6 +64,11 @@ postgres: table: points1_vw srid: 4326 geometry_column: geom + bounds: + - -179.27313970132585 + - -67.52518563265659 + - 162.60117193735186 + - 84.93092095128937 geometry_type: POINT properties: gid: int4 @@ -52,6 +77,11 @@ postgres: table: points2 srid: 4326 geometry_column: geom + bounds: + - -174.050750735362 + - -80.46177157848345 + - 179.11187181086706 + - 81.13068764165727 geometry_type: POINT properties: gid: int4 @@ -60,6 +90,11 @@ postgres: table: points3857 srid: 3857 geometry_column: geom + bounds: + - -161.40590777554058 + - -81.50727021609012 + - 172.51549126768532 + - 84.2440187164111 geometry_type: POINT properties: gid: int4 @@ -68,6 +103,11 @@ postgres: table: points_empty_srid srid: 900913 geometry_column: geom + bounds: + - -162.35196679784573 + - -84.49919770031491 + - 178.47294677445652 + - 82.7000012450467 geometry_type: GEOMETRY properties: gid: int4 @@ -76,6 +116,11 @@ postgres: table: table_source srid: 4326 geometry_column: geom + bounds: + - -2.0 + - -1.0 + - 142.84131509869133 + - 45.0 geometry_type: GEOMETRY properties: gid: int4 @@ -84,6 +129,11 @@ postgres: table: table_source_multiple_geom srid: 4326 geometry_column: geom1 + bounds: + - -136.62076049706184 + - -78.3350299285405 + - 176.56297743499888 + - 75.78731065954437 geometry_type: POINT properties: gid: int4 @@ -92,6 +142,11 @@ postgres: table: table_source_multiple_geom srid: 4326 geometry_column: geom2 + bounds: + - -136.62076049706184 + - -78.3350299285405 + - 176.56297743499888 + - 75.78731065954437 geometry_type: POINT properties: gid: int4 diff --git a/tests/test.sh b/tests/test.sh index 7cff1b6c0..2ff631790 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -153,7 +153,7 @@ echo "Test auto configured Martin" TEST_OUT_DIR="$(dirname "$0")/output/auto" mkdir -p "$TEST_OUT_DIR" -ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles) +ARG=(--default-srid 900913 --auto-bounds calc --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/mbtiles tests/fixtures/pmtiles) set -x $MARTIN_BIN "${ARG[@]}" 2>&1 | tee "${TMP_DIR}/test_log_1.txt" & PROCESS_ID=`jobs -p`