From f2bcf920fc2a9f8b798055775dd016d2481fb7ec Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 29 Sep 2023 18:46:22 -0400 Subject: [PATCH] mbtiles: remove tls features, CI streamlining --- .github/workflows/ci.yml | 26 +++--- Cargo.lock | 152 +----------------------------------- Cargo.toml | 4 +- martin-mbtiles/Cargo.toml | 14 ++-- martin-mbtiles/README.md | 17 +++- martin-mbtiles/src/lib.rs | 13 +-- martin-tile-utils/README.md | 15 ++++ martin/Cargo.toml | 2 +- 8 files changed, 61 insertions(+), 182 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39678e3cb..f7a8c6d94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,9 +58,8 @@ jobs: set -x cargo fmt --all -- --check cargo clippy --package martin-tile-utils -- -D warnings + cargo clippy --package martin-mbtiles --no-default-features -- -D warnings cargo clippy --package martin-mbtiles -- -D warnings - cargo clippy --package martin-mbtiles --no-default-features --features native-tls -- -D warnings - cargo clippy --package martin-mbtiles --no-default-features --features rustls -- -D warnings cargo clippy --package martin -- -D warnings cargo clippy --package martin --features vendored-openssl -- -D warnings cargo clippy --package martin --features bless-tests -- -D warnings @@ -73,16 +72,16 @@ jobs: - name: Build (native) if: matrix.cross != 'true' run: | - cargo build --release --target ${{ matrix.target }} --features=ssl --package martin - cargo build --release --target ${{ matrix.target }} --features=cli --package martin-mbtiles + cargo build --release --target ${{ matrix.target }} --package martin --features=ssl --package martin + cargo build --release --target ${{ matrix.target }} --package martin-mbtiles - name: Build (cross - aarch64-apple-darwin) if: matrix.target == 'aarch64-apple-darwin' run: | rustup target add "${{ matrix.target }}" # compile without debug symbols because stripping them with `strip` does not work cross-platform export RUSTFLAGS='-C link-arg=-s' - cargo build --release --target ${{ matrix.target }} --features=vendored-openssl --package martin - cargo build --release --target ${{ matrix.target }} --no-default-features --features=rustls,cli --package martin-mbtiles + cargo build --release --target ${{ matrix.target }} --package martin --features=vendored-openssl + cargo build --release --target ${{ matrix.target }} --package martin-mbtiles - name: Build (cross - aarch64-unknown-linux-gnu) if: matrix.target == 'aarch64-unknown-linux-gnu' run: | @@ -90,8 +89,8 @@ jobs: rustup target add "${{ matrix.target }}" # compile without debug symbols because stripping them with `strip` does not work cross-platform export RUSTFLAGS='-C link-arg=-s -C linker=aarch64-linux-gnu-gcc' - cargo build --release --target ${{ matrix.target }} --features=vendored-openssl --package martin - cargo build --release --target ${{ matrix.target }} --no-default-features --features=rustls,cli --package martin-mbtiles + cargo build --release --target ${{ matrix.target }} --package martin --features=vendored-openssl + cargo build --release --target ${{ matrix.target }} --package martin-mbtiles - name: Build (debian package) if: matrix.target == 'debian-x86_64' run: | @@ -152,10 +151,12 @@ jobs: run: | set -x cargo test --package martin-tile-utils + cargo test --package martin-mbtiles --no-default-features cargo test --package martin-mbtiles - cargo test --package martin-mbtiles --no-default-features --features rustls + cargo test --package martin cargo test --package martin --features vendored-openssl cargo test --doc + RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace cargo clean env: DATABASE_URL: ${{ steps.pg.outputs.connection-uri }} @@ -297,12 +298,7 @@ jobs: echo "Same but as base64 to prevent GitHub obfuscation (this is not a secret):" echo "$DATABASE_URL" | base64 set -x - cargo test --package martin-tile-utils - cargo test --package martin-mbtiles - cargo test --package martin-mbtiles --no-default-features --features rustls - cargo test --package martin --features vendored-openssl - cargo test --doc - RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace + cargo test --package martin cargo clean env: DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }} diff --git a/Cargo.lock b/Cargo.lock index 6b5ff6238..33a2cde97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -687,22 +687,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - [[package]] name = "cpufeatures" version = "0.2.9" @@ -1794,7 +1778,7 @@ dependencies = [ [[package]] name = "martin-mbtiles" -version = "0.4.0" +version = "0.5.0" dependencies = [ "actix-rt", "anyhow", @@ -1903,24 +1887,6 @@ dependencies = [ "serde", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -2047,12 +2013,6 @@ dependencies = [ "syn 2.0.37", ] -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - [[package]] name = "openssl-src" version = "300.1.5+3.1.3" @@ -2539,21 +2499,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted", - "web-sys", - "winapi", -] - [[package]] name = "roxmltree" version = "0.18.0" @@ -2622,9 +2567,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.14" +version = "0.38.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +checksum = "d2f9da0cbd88f9f09e7814e388301c8414c51c62aa6ce1e4b5c551d49d96e531" dependencies = [ "bitflags 2.4.0", "errno", @@ -2633,36 +2578,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rustls" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" -dependencies = [ - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" -dependencies = [ - "base64", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustybuzz" version = "0.7.0" @@ -2694,54 +2609,12 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" -dependencies = [ - "windows-sys", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "security-framework" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "1.0.19" @@ -3028,12 +2901,9 @@ dependencies = [ "indexmap 2.0.2", "log", "memchr", - "native-tls", "once_cell", "paste", "percent-encoding", - "rustls", - "rustls-pemfile", "serde", "serde_json", "sha2", @@ -3044,7 +2914,6 @@ dependencies = [ "tokio-stream", "tracing", "url", - "webpki-roots", ] [[package]] @@ -3675,12 +3544,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "url" version = "2.4.1" @@ -3863,15 +3726,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" -dependencies = [ - "rustls-webpki", -] - [[package]] name = "weezl" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index acc78dcba..9cc4e70e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ indoc = "2" itertools = "0.11" json-patch = "1.1" log = "0.4" -martin-mbtiles = { path = "./martin-mbtiles", version = "0.4.0", default-features = false, features = ["native-tls"] } # disable CLI tools +martin-mbtiles = { path = "./martin-mbtiles", version = "0.5.0", default-features = false } martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" } num_cpus = "1" openssl = "0.10" @@ -47,7 +47,7 @@ serde_json = "1" serde_yaml = "0.9" spreet = { version = "0.8", default-features = false } sqlite-hashes = "0.3" -sqlx = { version = "0.7", features = ["sqlite"] } +sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio"] } subst = { version = "0.3", features = ["yaml"] } thiserror = "1" tilejson = "0.3" diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index fd746f043..ef6a59111 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "martin-mbtiles" -version = "0.4.0" +version = "0.5.0" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] @@ -10,12 +10,8 @@ repository.workspace = true rust-version.workspace = true [features] -# TODO: Disable "cli" feature in default builds -default = ["cli", "native-tls"] -cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:tokio"] -# One of the following two must be used -native-tls = ["sqlx/runtime-tokio-native-tls"] -rustls = ["sqlx/runtime-tokio-rustls"] +default = ["cli"] +cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio"] [dependencies] futures.workspace = true @@ -23,6 +19,7 @@ log.workspace = true martin-tile-utils.workspace = true serde.workspace = true serde_json.workspace = true +sqlite-hashes.workspace = true sqlx.workspace = true thiserror.workspace = true tilejson.workspace = true @@ -31,8 +28,7 @@ tilejson.workspace = true anyhow = { workspace = true, optional = true } clap = { workspace = true, optional = true } env_logger = { workspace = true, optional = true } -serde_yaml.workspace = true -sqlite-hashes.workspace = true +serde_yaml = { workspace = true, optional = true } tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } [dev-dependencies] diff --git a/martin-mbtiles/README.md b/martin-mbtiles/README.md index d0a0484fd..04f8f5a7b 100644 --- a/martin-mbtiles/README.md +++ b/martin-mbtiles/README.md @@ -7,10 +7,25 @@ [![crates.io version](https://img.shields.io/crates/v/martin-mbtiles.svg)](https://crates.io/crates/martin-mbtiles) [![CI build](https://github.com/maplibre/martin/workflows/CI/badge.svg)](https://github.com/maplibre/martin-mbtiles/actions) -A library to help tile servers like [Martin](https://maplibre.org/martin) work with [MBTiles](https://github.com/mapbox/mbtiles-spec) files. +A library to help tile servers like [Martin](https://maplibre.org/martin) work with [MBTiles](https://github.com/mapbox/mbtiles-spec) files. When using as a lib, you may want to disable default features (i.e. the unused "cli" feature). This crate also has a small utility that allows users to interact with the `*.mbtiles` files from the command line. See [tools](https://maplibre.org/martin/tools.html) documentation for more information. ### Development Any changes to SQL commands require running of `just prepare-sqlite`. This will install `cargo sqlx` command if it is not already installed, and update the `./sqlx-data.json` file. + +## License + +Licensed under either of + +* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) + at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the +Apache-2.0 license, shall be dual licensed as above, without any +additional terms or conditions. diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index 16a9747dc..65522b564 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -1,12 +1,15 @@ #![allow(clippy::missing_errors_doc)] mod errors; -mod mbtiles; -mod mbtiles_pool; -mod mbtiles_queries; -mod tile_copier; - pub use errors::{MbtError, MbtResult}; + +mod mbtiles; pub use mbtiles::{IntegrityCheckType, Mbtiles, Metadata}; + +mod mbtiles_pool; pub use mbtiles_pool::MbtilesPool; + +mod tile_copier; pub use tile_copier::{apply_mbtiles_diff, CopyDuplicateMode, TileCopierOptions}; + +mod mbtiles_queries; diff --git a/martin-tile-utils/README.md b/martin-tile-utils/README.md index 813c13ec4..2bdea051d 100644 --- a/martin-tile-utils/README.md +++ b/martin-tile-utils/README.md @@ -7,3 +7,18 @@ [![CI build](https://github.com/maplibre/martin/workflows/CI/badge.svg)](https://github.com/maplibre/martin-tile-utils/actions) A library to help tile servers like [Martin](https://maplibre.org/martin) work with tile content. + +## License + +Licensed under either of + +* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) + at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the +Apache-2.0 license, shall be dual licensed as above, without any +additional terms or conditions. diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 31bd47f4b..7127dc286 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -2,7 +2,7 @@ name = "martin" # Make sure to update /home/nyurik/dev/rust/martin/homebrew-formula/martin.rb version # Once the release is published with the hash -version = "0.9.0-pre.1" +version = "0.9.0-pre.3" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"]