From 9f9f18163c2c224efc1ff5ccd4119e9655a4ae27 Mon Sep 17 00:00:00 2001 From: Josh Lee Date: Thu, 1 Feb 2024 13:56:40 -0500 Subject: [PATCH] Add AWS Lambda support (#1127) This adds the lambda-web crate to adapt the actix App to speak to Lambda by way of the lambda_runtime crate. AWS Lambda has native support for scripting languages to execute a function directly; compiled languages must embed a runtime to fetch incoming events from Lambda and post the responses. This detects the environment variables to start up in Lambda mode instead of the normal HTTP server, and is added as an optional feature. Lambda has five (!) distinct ways of routing HTTP requests to a function; this supports some of them. (Specifically, the most obvious way to do this is with a Function URL, which is newest and simplest, and perhaps with CloudFront, which speaks to the Function URL and not Lambda directly.) The error handling could probably be refined, I was just trying to get this to compile. (Supported: API Gateway HTTP API with payload format version 2.0; API Gateway REST API; Lambda function URLs / Not supported: API Gateway HTTP API with payload format version 1.0; Application Load Balancer) Necessary for #1102 to be able to run the released packages directly, and only having to configure the appropriate environment. --------- Co-authored-by: Yuri Astrakhan --- .dockerignore | 1 + .github/files/lambda-function/Makefile | 2 + .github/files/lambda-function/config.yaml | 3 + .github/files/lambda-layer/bootstrap | 3 + .github/files/lambda.yaml | 19 +++ .github/workflows/ci.yml | 21 ++++ .gitignore | 1 + Cargo.lock | 139 ++++++++++++++++++++-- Cargo.toml | 1 + docs/src/SUMMARY.md | 1 + docs/src/env-vars.md | 1 + docs/src/run-with-lambda.md | 131 ++++++++++++++++++++ justfile | 4 + martin/Cargo.toml | 4 +- martin/src/bin/martin-cp.rs | 20 ++-- martin/src/bin/martin.rs | 30 ++--- martin/src/srv/server.rs | 49 +++++--- martin/src/utils/error.rs | 5 +- tests/test-aws-lambda.sh | 34 ++++++ 19 files changed, 408 insertions(+), 61 deletions(-) create mode 100644 .github/files/lambda-function/Makefile create mode 100644 .github/files/lambda-function/config.yaml create mode 100755 .github/files/lambda-layer/bootstrap create mode 100644 .github/files/lambda.yaml create mode 100644 docs/src/run-with-lambda.md create mode 100755 tests/test-aws-lambda.sh diff --git a/.dockerignore b/.dockerignore index 39dc26852..05147945c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,3 +23,4 @@ pg_data/ config.yml tests/output/ tmp/ +.aws-sam/ diff --git a/.github/files/lambda-function/Makefile b/.github/files/lambda-function/Makefile new file mode 100644 index 000000000..1dfa25496 --- /dev/null +++ b/.github/files/lambda-function/Makefile @@ -0,0 +1,2 @@ +build-MartinFunction: + cp -a . $(ARTIFACTS_DIR) diff --git a/.github/files/lambda-function/config.yaml b/.github/files/lambda-function/config.yaml new file mode 100644 index 000000000..36cdfa5c9 --- /dev/null +++ b/.github/files/lambda-function/config.yaml @@ -0,0 +1,3 @@ +pmtiles: + sources: + webp2: ./webp2.pmtiles diff --git a/.github/files/lambda-layer/bootstrap b/.github/files/lambda-layer/bootstrap new file mode 100755 index 000000000..1b51ae350 --- /dev/null +++ b/.github/files/lambda-layer/bootstrap @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec martin -c "${_HANDLER}" diff --git a/.github/files/lambda.yaml b/.github/files/lambda.yaml new file mode 100644 index 000000000..7baaf448a --- /dev/null +++ b/.github/files/lambda.yaml @@ -0,0 +1,19 @@ +# This is a minimal AWS SAM template sufficient to invoke the handler +# in the CI environment. It probably hasn't been tested for actual +# deployments. + +AWSTemplateFormatVersion: 2010-09-09 +Transform: 'AWS::Serverless-2016-10-31' +Resources: + MartinLayer: + Type: 'AWS::Serverless::LayerVersion' + Properties: + ContentUri: lambda-layer/ + MartinFunction: + Type: 'AWS::Serverless::Function' + Properties: + Runtime: provided.al2023 + Layers: + - Ref: MartinLayer + CodeUri: lambda-function/ + Handler: config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cb120859..5e73ec00b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,10 @@ jobs: with: install: true platforms: linux/amd64,linux/arm64 + - name: Set up AWS SAM + uses: aws-actions/setup-sam@v2 + with: + use-installer: true - name: Build targets run: | @@ -177,6 +181,7 @@ jobs: load: true tags: ${{ github.repository }}:linux-arm64 platforms: linux/arm64 + - name: Test linux/arm64 Docker image run: | PLATFORM=linux/arm64 @@ -288,6 +293,22 @@ jobs: name: build-${{ matrix.target }} path: target_releases/* + test-aws-lambda: + name: Test AWS Lambda + runs-on: ubuntu-latest + needs: [ build ] + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - name: Download build artifact cross-build + uses: actions/download-artifact@v3 + with: + name: cross-build + - run: tests/test-aws-lambda.sh + env: + MARTIN_BIN: x86_64-unknown-linux-musl/martin + + test-multi-os: name: Test on ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 80282051e..6dec1c8b4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ pg_data/ config.yml tests/output/ tmp/ +.aws-sam/ diff --git a/Cargo.lock b/Cargo.lock index d88b13461..723c65d6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash", - "base64", + "base64 0.21.7", "bitflags 2.4.2", "brotli", "bytes", @@ -387,6 +387,28 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "async-trait" version = "0.1.77" @@ -438,6 +460,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -1920,6 +1948,55 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "lambda-web" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6277b60649250d681654162b7e8e875c938295ea5f883eb9a8da7e27d2c051" +dependencies = [ + "actix-http", + "actix-service", + "actix-web", + "base64 0.13.1", + "brotli", + "lambda_runtime", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "lambda_runtime" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd32d5799db2155ae4d47116bb3e169b59f531ced4d5762a10c2125bdd2bf134" +dependencies = [ + "async-stream", + "bytes", + "futures", + "http", + "hyper", + "lambda_runtime_api_client", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "lambda_runtime_api_client" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7210012be904051520f0dc502140ba599bae3042b65b3737b87727f1aa88a7d6" +dependencies = [ + "http", + "hyper", + "tokio", + "tower-service", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -2044,6 +2121,7 @@ dependencies = [ "insta", "itertools 0.12.1", "json-patch", + "lambda-web", "log", "martin-tile-utils", "mbtiles", @@ -2478,7 +2556,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64", + "base64 0.21.7", "serde", ] @@ -2521,6 +2599,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -2653,7 +2751,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator 0.2.0", @@ -3005,7 +3103,7 @@ version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -3249,7 +3347,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -3258,7 +3356,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" dependencies = [ - "base64", + "base64 0.21.7", "rustls-pki-types", ] @@ -3457,7 +3555,7 @@ version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b0ed1662c5a68664f45b76d18deb0e234aff37207086803165c961eb695e981" dependencies = [ - "base64", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -3784,7 +3882,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" dependencies = [ "atoi", - "base64", + "base64 0.21.7", "bitflags 2.4.2", "byteorder", "bytes", @@ -3826,7 +3924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" dependencies = [ "atoi", - "base64", + "base64 0.21.7", "bitflags 2.4.2", "byteorder", "crc", @@ -4305,6 +4403,27 @@ dependencies = [ "serde", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -4495,7 +4614,7 @@ version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51daa774fe9ee5efcf7b4fec13019b8119cda764d9a8b5b06df02bb1445c656" dependencies = [ - "base64", + "base64 0.21.7", "log", "pico-args", "usvg-parser", diff --git a/Cargo.toml b/Cargo.toml index deb53b7c0..daf579550 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ indoc = "2" insta = "1" itertools = "0.12" json-patch = "1.2" +lambda-web = { version = "0.2.1", features = ["actix4"] } log = "0.4" martin-tile-utils = { path = "./martin-tile-utils", version = "0.4.0" } mbtiles = { path = "./mbtiles", version = "0.9.0" } diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index ccb584208..bd7d57b55 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -7,6 +7,7 @@ - [Running with Docker](run-with-docker.md) - [Running with Docker Compose](run-with-docker-compose.md) - [Running with NGINX](run-with-nginx.md) + - [Running in AWS Lambda](run-with-lambda.md) - [Troubleshooting](troubleshooting.md) - [Configuration File](config-file.md) - [PostgreSQL Connections](pg-connections.md) diff --git a/docs/src/env-vars.md b/docs/src/env-vars.md index 0c9ef7536..0bb98a9e9 100644 --- a/docs/src/env-vars.md +++ b/docs/src/env-vars.md @@ -9,3 +9,4 @@ You can also configure Martin using environment variables, but only if the confi | `PGSSLCERT`
`ssl_cert` | `./postgresql.crt` | A file with a client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT) | | `PGSSLKEY`
`ssl_key` | `./postgresql.key` | A file with the key for the client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY) | | `PGSSLROOTCERT`
`ssl_root_cert` | `./root.crt` | A file with trusted root certificate(s). The file should contain a sequence of PEM-formatted CA certificates. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT) | +| `AWS_LAMBDA_RUNTIME_API` | | If defined, connect to AWS Lambda to handle requests. The regular HTTP server is not used. See [Running in AWS Lambda](run-with-lambda.md) | diff --git a/docs/src/run-with-lambda.md b/docs/src/run-with-lambda.md new file mode 100644 index 000000000..59641418a --- /dev/null +++ b/docs/src/run-with-lambda.md @@ -0,0 +1,131 @@ +## Using with AWS Lambda + +Martin can be run in AWS Lambda. This is useful if you want to serve tiles from a serverless environment, while accessing "nearby" data from a PostgreSQL database or PMTiles file in S3, without exposing the raw file to the world to prevent download abuse and improve performance. + +Some very brief context: Lambda has two deployment models, zip file and container-based. When using zip file deployment, the online code editor is available, in which we can edit the .yaml configuration. When using container-based deployment, we can pass our configuration on the command line or environment variables. + +Everything can be performed from AWS CloudShell, otherwise you will need to install the AWS CLI and the AWS SAM CLI, and configure authentication. The CloudShell also runs in a particular AWS region. + +### Container deployment + +Lambda images must come from a public or private ECR registry. Pull the image from GHCR and push it to ECR. + +```bash +$ docker pull ghcr.io/maplibre/martin:latest --platform linux/arm64 +$ aws ecr create-repository --repository-name martin +[…] + "repositoryUri": "493749042871.dkr.ecr.us-east-2.amazonaws.com/martin", +# Read the repositoryUri which includes your account number +$ docker tag ghcr.io/maplibre/martin:latest 493749042871.dkr.ecr.us-east-2.amazonaws.com/martin:latest +$ aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 493749042871.dkr.ecr.us-east-2.amazonaws.com +$ docker push 493749042871.dkr.ecr.us-east-2.amazonaws.com/martin:latest +``` + +Now you can go to the [Lambda console](https://console.aws.amazon.com/lambda) and create your function. + +1. Click “Create function”. +2. Choose “Container image”. +3. Put something in “Function name”. (Note: This is an internal identifier, not exposed in the function URL.) +4. Click “Browse images”, and select your repository and the tag. (If you can’t find it, see if you’re in the same region?) +5. Expand “Container image overrides”, and under CMD put the URL of a .pmtiles file. +6. Set “Architecture” to arm64 to match the platform that we pulled. (Lambda has better ARM CPUs than x86.) +7. Click “Create function”. +8. Find the “Configuration” tab, select “Function URL”, “Create function URL”. +9. Set “Auth type” to `NONE` + * Do not enable CORS. Martin already has CORS support, so it will create duplicate headers and break CORS. +10. Click on the “Function URL”. If it works, hooray! If it doesn’t, open the “Monitor” tab, “View CloudWatch logs”, find the most recent Log stream. + +### Zip deployment + +It’s possible to deploy the entire codebase from the AWS console, but we will use Serverless Application Model. Our function will consist of a “Layer”, containing the Martin binary, and our function itself will contain the configuration in .yaml format. + +#### The layer + +Download the binary and place it in your staging directory. The `bin` directory of your Layer will be added to the PATH. + +```bash +mkdir -p martin_layer/src/bin/ +cd martin_layer +curl -OL https://github.com/maplibre/martin/releases/latest/download/martin-aarch64-unknown-linux-musl.tar.gz +tar -C src/bin/ -xzf martin-aarch64-unknown-linux-musl.tar.gz martin +``` + +Every zip-based Lambda function runs a file called `bootstrap`. + +```bash +cat <src/bootstrap +#!/bin/sh +set -eu +exec martin -c ${_HANDLER}.yaml +EOF +``` + +Write the SAM template. + +```yaml +cat <template.yaml +AWSTemplateFormatVersion: 2010-09-09 +Transform: 'AWS::Serverless-2016-10-31' +Resources: + martin: + Type: 'AWS::Serverless::LayerVersion' + DeletionPolicy: Delete + Properties: + ContentUri: src + CompatibleRuntimes: + - provided.al2023 + CompatibleArchitectures: + - arm64 +Outputs: + LayerArn: + Value: !Ref MartinLayer + Export: + Name: !Sub "${AWS::StackName}-LayerArn" +EOF +``` + +Run `sam deploy --guided`. + +1. Stack Name: Name your CloudFormation stack something like `martin-layer`. +2. Press enter for everything else +3. The settings are saved to `samconfig.toml`, so you can later do `sam deploy` to update the version, or `sam delete`. + +Now if you visit the [Lambda console](https://console.aws.amazon.com/lambda/home) and select “Layers”, you should see your layer. + +#### The function + +1. Select “Functions”, “Create function”. +2. Put something in “Function name”. +3. Set “Runtime” to “Amazon Linux 2023”. +4. Set “Architecture” to “arm64”. +5. Under “Advanced settings”, choose “Enable function URL” with “Auth type” of “NONE”. +6. Click “Create function”. + +Add your layer: + +1. Click “add a layer” (green banner at the top, or the very bottom). +2. Choose “Custom layers”, and select your layer and its version. +3. Click “Add”. + +Add your configuration file in the function source code: + +1. Code tab, File, New File: `hello.handler.yaml`. + + ```yaml + pmtiles: + sources: + demotiles: + ``` + +2. Click Deploy, wait for the success banner, and visit your function URL. + +### TODO + +This support is preliminary; there are features to add to Martin, configuration to tweak, and documentation to write. + +* Lambda has a default timeout of 3 seconds, and 128 MB of memory, maybe this is suboptimal. +* Document how to connect to a PostgreSQL database on RDS. +* Set up a CloudFront CDN, this is a whole thing, but explain the motivation and the basics. +* Grant the execution role permission to read objects from an S3 bucket, and teach Martin how to make authenticated requests to S3. +* Teach Martin how to serve all PMTiles files from an S3 bucket rather than having to list them at startup. +* Teach Martin how to set the Cache-Control and Etag headers for better defaults. diff --git a/justfile b/justfile index f92dc9bf1..ac5576d69 100644 --- a/justfile +++ b/justfile @@ -177,6 +177,10 @@ test-int: clean-test install-sqlx fi fi +# Run AWS Lambda smoke test against SAM local +test-lambda: + tests/test-aws-lambda.sh + # Run integration tests and save its output as the new expected output bless: restart clean-test bless-insta-martin bless-insta-mbtiles bless-tests bless-int diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 24a778ca7..91d3f0a6f 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -60,8 +60,9 @@ name = "bench" harness = false [features] -default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] +default = ["fonts", "lambda", "mbtiles", "pmtiles", "postgres", "sprites"] fonts = ["dep:bit-set", "dep:pbf_font_tools"] +lambda = ["dep:lambda-web"] mbtiles = ["dep:mbtiles"] pmtiles = ["dep:pmtiles", "dep:reqwest"] postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] @@ -110,6 +111,7 @@ tilejson.workspace = true tokio = { workspace = true, features = ["io-std"] } tokio-postgres-rustls = { workspace = true, optional = true } url.workspace = true +lambda-web = { workspace = true, optional = true } [dev-dependencies] cargo-husky.workspace = true diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index feecbfc95..d24031ea9 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -430,19 +430,15 @@ async fn main() { let env = env_logger::Env::default().default_filter_or("martin_cp=info"); env_logger::Builder::from_env(env).init(); - start(CopierArgs::parse()) - .await - .unwrap_or_else(|e| on_error(e)); -} - -fn on_error(e: E) -> ! { - // Ensure the message is printed, even if the logging is disabled - if log_enabled!(log::Level::Error) { - error!("{e}"); - } else { - eprintln!("{e}"); + if let Err(e) = start(CopierArgs::parse()).await { + // Ensure the message is printed, even if the logging is disabled + if log_enabled!(log::Level::Error) { + error!("{e}"); + } else { + eprintln!("{e}"); + } + std::process::exit(1); } - std::process::exit(1); } #[cfg(test)] diff --git a/martin/src/bin/martin.rs b/martin/src/bin/martin.rs index 054970e49..d54adeba5 100644 --- a/martin/src/bin/martin.rs +++ b/martin/src/bin/martin.rs @@ -1,6 +1,3 @@ -use std::fmt::Display; - -use actix_web::dev::Server; use clap::Parser; use log::{error, info, log_enabled}; use martin::args::{Args, OsEnv}; @@ -9,7 +6,7 @@ use martin::{read_config, Config, MartinResult}; const VERSION: &str = env!("CARGO_PKG_VERSION"); -async fn start(args: Args) -> MartinResult { +async fn start(args: Args) -> MartinResult<()> { info!("Starting Martin v{VERSION}"); let env = OsEnv::default(); @@ -35,8 +32,7 @@ async fn start(args: Args) -> MartinResult { let (server, listen_addresses) = new_server(config.srv, sources)?; info!("Martin has been started on {listen_addresses}."); info!("Use http://{listen_addresses}/catalog to get the list of available sources."); - - Ok(server) + server.await } #[actix_web::main] @@ -44,19 +40,13 @@ async fn main() { let env = env_logger::Env::default().default_filter_or("martin=info"); env_logger::Builder::from_env(env).init(); - start(Args::parse()) - .await - .unwrap_or_else(|e| on_error(e)) - .await - .unwrap_or_else(|e| on_error(e)); -} - -fn on_error(e: E) -> ! { - // Ensure the message is printed, even if the logging is disabled - if log_enabled!(log::Level::Error) { - error!("{e}"); - } else { - eprintln!("{e}"); + if let Err(e) = start(Args::parse()).await { + // Ensure the message is printed, even if the logging is disabled + if log_enabled!(log::Level::Error) { + error!("{e}"); + } else { + eprintln!("{e}"); + } + std::process::exit(1); } - std::process::exit(1); } diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index 79d87262e..cf89cc605 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -1,13 +1,17 @@ +use std::future::Future; +use std::pin::Pin; use std::string::ToString; use std::time::Duration; use actix_cors::Cors; -use actix_web::dev::Server; use actix_web::error::ErrorInternalServerError; use actix_web::http::header::CACHE_CONTROL; use actix_web::middleware::TrailingSlash; use actix_web::web::Data; use actix_web::{middleware, route, web, App, HttpResponse, HttpServer, Responder}; +use futures::TryFutureExt; +#[cfg(feature = "lambda")] +use lambda_web::{is_running_on_lambda, run_actix_on_lambda}; use log::error; use serde::{Deserialize, Serialize}; @@ -98,16 +102,13 @@ pub fn router(cfg: &mut web::ServiceConfig) { cfg.service(crate::srv::fonts::get_font); } -/// Create a new initialized Actix `App` instance together with the listening address. +type Server = Pin>>>; + +/// Create a future for an Actix web server together with the listening address. pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server, String)> { let catalog = Catalog::new(&state)?; - let keep_alive = Duration::from_secs(config.keep_alive.unwrap_or(KEEP_ALIVE_DEFAULT)); - let worker_processes = config.worker_processes.unwrap_or_else(num_cpus::get); - let listen_addresses = config - .listen_addresses - .unwrap_or_else(|| LISTEN_ADDRESSES_DEFAULT.to_owned()); - let server = HttpServer::new(move || { + let factory = move || { let cors_middleware = Cors::default() .allow_any_origin() .allowed_methods(vec!["GET"]); @@ -127,15 +128,29 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server .wrap(middleware::NormalizePath::new(TrailingSlash::MergeOnly)) .wrap(middleware::Logger::default()) .configure(router) - }) - .bind(listen_addresses.clone()) - .map_err(|e| BindingError(e, listen_addresses.clone()))? - .keep_alive(keep_alive) - .shutdown_timeout(0) - .workers(worker_processes) - .run(); - - Ok((server, listen_addresses)) + }; + + #[cfg(feature = "lambda")] + if is_running_on_lambda() { + let server = run_actix_on_lambda(factory).err_into(); + return Ok((Box::pin(server), "(aws lambda)".into())); + } + + let keep_alive = Duration::from_secs(config.keep_alive.unwrap_or(KEEP_ALIVE_DEFAULT)); + let worker_processes = config.worker_processes.unwrap_or_else(num_cpus::get); + let listen_addresses = config + .listen_addresses + .unwrap_or_else(|| LISTEN_ADDRESSES_DEFAULT.to_owned()); + + let server = HttpServer::new(factory) + .bind(listen_addresses.clone()) + .map_err(|e| BindingError(e, listen_addresses.clone()))? + .keep_alive(keep_alive) + .shutdown_timeout(0) + .workers(worker_processes) + .run() + .err_into(); + Ok((Box::pin(server), listen_addresses)) } #[cfg(test)] diff --git a/martin/src/utils/error.rs b/martin/src/utils/error.rs index 989a078bb..2843eb452 100644 --- a/martin/src/utils/error.rs +++ b/martin/src/utils/error.rs @@ -80,6 +80,9 @@ pub enum MartinError { #[error(transparent)] WebError(#[from] actix_web::Error), + #[error(transparent)] + IoError(#[from] io::Error), + #[error("Internal error: {0}")] - InternalError(Box), + InternalError(#[from] Box), } diff --git a/tests/test-aws-lambda.sh b/tests/test-aws-lambda.sh new file mode 100755 index 000000000..7ac710a67 --- /dev/null +++ b/tests/test-aws-lambda.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +have () { + hash -- "$1" 2>&- +} + +if ! have sam; then + echo "The AWS Serverless Application Model Command Line Interface (AWS SAM CLI) " + echo "must be installed: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html" + exit 1 +fi + +# Just send a single request using `sam local invoke` to verify that +# the server boots, finds a source to serve, and can handle a request. +# TODO Run the fuller integration suite against this. +# In doing so, switch from `sam local invoke`, which starts and stops the +# server, to `sam local start-api`, which keeps it running. + +EVENT=$(sam local generate-event apigateway http-api-proxy \ + | jq '.rawPath="/"|.requestContext.http.method="GET"') + +# `sam build` will copy the _entire_ context to a temporary directory, +# so just give it the files we need +mkdir -p .github/files/lambda-layer/bin/ +if ! install ${MARTIN_BIN:-target/debug/martin} .github/files/lambda-layer/bin/; then + echo "Specify the binary, e.g. ‘MARTIN_BIN=target/x86_64-linux-unknown-musl/release/martin just test-lambda’" + exit 1 +fi +cp ./tests/fixtures/pmtiles2/webp2.pmtiles .github/files/lambda-function/ +sam build -t .github/files/lambda.yaml + +echo "$EVENT" | sam local invoke -e - \ + | jq -ne 'input.statusCode==200'