From a687c9132f29cf142b96d18ec921e906ab070a78 Mon Sep 17 00:00:00 2001 From: Jonathan Rosenberg <96974219+Jonathan-Rosenberg@users.noreply.github.com> Date: Sun, 3 Dec 2023 15:16:53 +0200 Subject: [PATCH] use a writer_client function instead of a storage client when passed to the export_delta_log function --- examples/hooks/delta_lake_S3_export.lua | 2 +- pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/hooks/delta_lake_S3_export.lua b/examples/hooks/delta_lake_S3_export.lua index d60d01e01fb..6899be15795 100644 --- a/examples/hooks/delta_lake_S3_export.lua +++ b/examples/hooks/delta_lake_S3_export.lua @@ -18,7 +18,7 @@ local delta_export = require("lakefs/catalogexport/delta_exporter") local sc = aws.s3_client(args.aws.aws_access_key_id, args.aws.aws_secret_access_key, args.aws.aws_region) local delta_client = delta_export.get_delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.aws_region) -local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc, delta_client) +local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc.put_object, delta_client) for t, loc in pairs(delta_table_locations) do print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n") end diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua index 4c9fadd1e72..c37b6139f84 100644 --- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua +++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua @@ -39,14 +39,13 @@ end table_paths: ["path/to/table1", "path/to/table2", ...] - storage_client: - - put_object: function(bucket, key, data) + write_object: function(bucket, key, data) delta_client: - get_table: function(repo, ref, prefix) ]] -local function export_delta_log(action, table_paths, storage_client, delta) +local function export_delta_log(action, table_paths, write_object, delta) local repo = action.repository_id local commit_id = action.commit_id @@ -129,7 +128,7 @@ local function export_delta_log(action, table_paths, storage_client, delta) table_entry_string = table_entry_string .. content_entry end local version_key = storage_props.key .. "/" .. entry_version - storage_client.put_object(storage_props.bucket, version_key, table_entry_string) + write_object(storage_props.bucket, version_key, table_entry_string) end response[t_name] = table_physical_path end