diff --git a/Cargo.toml b/Cargo.toml index ca0c5cd..37568d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,8 +20,8 @@ anyhow = "=1" chrono = "0.4.31" aws_lambda_events = { version = "0.15.1", default-features = false, features = ["sns", "sqs", "s3"] } # The datafusion feature is required to support invariants which may be in error, but is required as of currently released 0.18.2 -deltalake = { version = "0.22.0", features = ["s3", "json", "datafusion"]} -#deltalake = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["s3", "json", "datafusion"]} +#deltalake = { version = "0.21.0", features = ["s3", "json", "datafusion"]} +deltalake = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["s3", "json", "datafusion"]} #deltalake = { path = "../../delta-io/delta-rs/crates/deltalake", features = ["s3", "json", "datafusion"]} tokio = { version = "=1", features = ["macros"] } regex = "=1" diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 0000000..37c541a --- /dev/null +++ b/Cross.toml @@ -0,0 +1,11 @@ + +# This is currently needed to ensure openssl libraries are bundled into the +# cross compilation image +# +# This is due to the reqwest library being pulled in recent releases with +# default features enabled, including native-tls :( +[target.x86_64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH" +] diff --git a/ci/build-release.sh b/ci/build-release.sh index df38d55..07ddd3e 100755 --- a/ci/build-release.sh +++ b/ci/build-release.sh @@ -6,4 +6,4 @@ fi; . venv/bin/activate -exec cargo lambda build --release --output-format zip +exec cargo lambda build --compiler cross --release --output-format zip diff --git a/lambdas/file-loader/src/main.rs b/lambdas/file-loader/src/main.rs index 2d67d3e..58ca572 100644 --- a/lambdas/file-loader/src/main.rs +++ b/lambdas/file-loader/src/main.rs @@ -94,13 +94,6 @@ async fn main() -> Result<(), Error> { let _ = env::var("DELTA_TABLE_URI").expect("The `DELTA_TABLE_URI` must be set in the environment"); - match env::var("DYNAMO_LOCK_TABLE_NAME") { - Ok(_) => {} - Err(_) => { - warn!("file-loader SHOULD have `DYNAMO_LOCK_TABLE_NAME` set to a valid name, and should have AWS_S3_LOCKING_PROVIDER=dynamodb set so that concurrent writes can be performed safely."); - } - } - info!("Starting file-loader"); run(service_fn(function_handler)).await