From c446901de89eb24b05dc263c8798125bdbe52b6d Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 23 Nov 2023 15:50:44 +0200
Subject: [PATCH 01/35] delta client impl

---
 go.mod                                        |  20 +-
 pkg/actions/lua/encoding/json/json.go         |  43 ++++-
 pkg/actions/lua/formats/delta.go              | 178 ++++++++++++++++++
 pkg/actions/lua/formats/formats.go            |  17 ++
 .../lakefs/catalogexport/delta_exporter.lua   |  49 +++++
 .../lua/lakefs/catalogexport/internal.lua     |  28 ++-
 .../lakefs/catalogexport/storage_utils.lua    |  34 ++++
 7 files changed, 360 insertions(+), 9 deletions(-)
 create mode 100644 pkg/actions/lua/formats/delta.go
 create mode 100644 pkg/actions/lua/formats/formats.go
 create mode 100644 pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
 create mode 100644 pkg/actions/lua/lakefs/catalogexport/storage_utils.lua

diff --git a/go.mod b/go.mod
index 2c55d82d2e6..97183e2155f 100644
--- a/go.mod
+++ b/go.mod
@@ -102,7 +102,10 @@ require (
 	cloud.google.com/go/iam v1.1.2 // indirect
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
+	github.com/ahmetb/go-linq/v3 v3.2.0 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
@@ -118,13 +121,18 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
+	github.com/barweiss/go-tuple v1.1.1 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/csimplestring/delta-go v0.0.0-20231105162402-9b93ca02cedf // indirect
+	github.com/deckarep/golang-set/v2 v2.3.1 // indirect
+	github.com/fraugster/parquet-go v0.12.0 // indirect
 	github.com/getsentry/sentry-go v0.16.0 // indirect
-	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/glog v1.1.0 // indirect
 	github.com/google/flatbuffers v2.0.0+incompatible // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
@@ -133,17 +141,25 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/oklog/run v1.0.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.8 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/repeale/fp-go v0.11.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	github.com/rotisserie/eris v0.5.4 // indirect
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/samber/mo v1.10.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
+	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
+	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	gocloud.dev v0.34.0 // indirect
 	gonum.org/v1/gonum v0.9.3 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect
diff --git a/pkg/actions/lua/encoding/json/json.go b/pkg/actions/lua/encoding/json/json.go
index 6c988c127a5..d8973eac493 100644
--- a/pkg/actions/lua/encoding/json/json.go
+++ b/pkg/actions/lua/encoding/json/json.go
@@ -3,6 +3,7 @@ package json
 import (
 	"bytes"
 	"encoding/json"
+	"reflect"
 
 	"github.com/Shopify/go-lua"
 	"github.com/treeverse/lakefs/pkg/actions/lua/util"
@@ -31,20 +32,60 @@ func check(l *lua.State, err error) {
 
 func jsonMarshal(l *lua.State) int {
 	var t interface{}
+	var ot interface{}
 	var err error
 	if !l.IsNil(1) {
 		t, err = util.PullTable(l, 1)
 		check(l, err)
 	}
+	if !l.IsNoneOrNil(2) { // Options table
+		ot, err = util.PullTable(l, 2)
+		check(l, err)
+	}
 	var buf bytes.Buffer
 	e := json.NewEncoder(&buf)
-	e.SetIndent("", "  ")
+	var rot map[string]any
+	if ot != nil {
+		vot := reflect.ValueOf(ot)
+		rot = vot.Interface().(map[string]any)
+	}
+	if rot != nil {
+		if pre, ind, ok := fetchIndentProps(rot); ok {
+			e.SetIndent(pre, ind)
+		}
+	}
+	//e.SetIndent("", "  ")
 	err = e.Encode(t)
 	check(l, err)
 	l.PushString(buf.String())
 	return 1
 }
 
+func fetchIndentProps(rot map[string]any) (string, string, bool) {
+	var prefix *string
+	var indent *string
+	if p, ok := rot["prefix"]; ok {
+		sp := p.(string)
+		prefix = &sp
+	}
+	if i, ok := rot["indent"]; ok {
+		si := i.(string)
+		indent = &si
+	}
+	if prefix != nil || indent != nil {
+		if prefix == nil {
+			defaultPref := ""
+			prefix = &defaultPref
+		}
+		if indent == nil {
+			defaultIndent := ""
+			indent = &defaultIndent
+		}
+		return *prefix, *indent, true
+	}
+	return "", "", false
+}
+
 func jsonUnmarshal(l *lua.State) int {
 	payload := lua.CheckString(l, 1)
 	var output interface{}
diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
new file mode 100644
index 00000000000..210128de619
--- /dev/null
+++ b/pkg/actions/lua/formats/delta.go
@@ -0,0 +1,178 @@
+package formats
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/Shopify/go-lua"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	delta "github.com/csimplestring/delta-go"
+	deltaStore "github.com/csimplestring/delta-go/store"
+	luautil "github.com/treeverse/lakefs/pkg/actions/lua/util"
+	"path"
+)
+
+type storageType string
+
+const (
+	s3StorageType    storageType = "s3"
+	gcsStorageType   storageType = "gcs"
+	azureStorageType storageType = "azure"
+)
+
+type DeltaClient struct {
+	accessProvider AccessProvider
+	ctx            context.Context
+}
+
+func (dc *DeltaClient) s3Fetcher(repo, ref, prefix string) (map[int64][]string, error) {
+	table, err := dc.buildS3DeltaTable(repo, ref, prefix)
+	if err != nil {
+		return nil, err
+	}
+	return dc.buildLog(table)
+}
+func (dc *DeltaClient) buildS3DeltaTable(repo, ref, prefix string) (delta.Log, error) {
+	awsInfo := dc.accessProvider.(AWSInfo)
+	config := delta.Config{StoreType: string(s3StorageType)}
+	s3LogStore, err := deltaStore.NewS3CompatLogStore(&awsInfo.AWSProps, repo, path.Join(ref, prefix))
+	if err != nil {
+		return nil, err
+	}
+	store := deltaStore.Store(s3LogStore)
+	url := fmt.Sprintf("lakefs://%s/%s/%s", repo, ref, prefix)
+	return delta.ForTableWithStore(url, config, &delta.SystemClock{}, &store)
+}
+func (dc *DeltaClient) buildLog(table delta.Log) (map[int64][]string, error) {
+	//s, err := table.Snapshot()
+	//if err != nil {
+	//	return nil, err
+	//}
+	//version := s.Version()
+
+	//versionLog, err := table.Changes(version, false)
+
+	versionLog, err := table.Changes(0, false)
+	if err != nil {
+		return nil, err
+	}
+
+	entries := make(map[int64][]string)
+	for entry, err := versionLog.Next(); err == nil; entry, err = versionLog.Next() {
+		strLog := make([]string, 0)
+		entryVersion := entry.Version()
+		actions, aErr := entry.Actions()
+		if aErr != nil {
+			return nil, err
+		}
+		for _, a := range actions {
+			aj, _ := a.Json()
+			strLog = append(strLog, aj)
+		}
+		entries[entryVersion] = strLog
+	}
+	return entries, nil
+}
+
+func (dc *DeltaClient) fetchTableLog(repo, ref, prefix string) (map[int64][]string, error) {
+	ap, _ := dc.accessProvider.GetAccessProperties()
+	switch ap.(type) {
+	case AWSInfo:
+		return dc.s3Fetcher(repo, ref, prefix)
+	default:
+		return nil, errors.New("unimplemented provider")
+	}
+
+}
+
+func getTable(client *DeltaClient) lua.Function {
+	return func(l *lua.State) int {
+		repo := lua.CheckString(l, 1)
+		ref := lua.CheckString(l, 2)
+		prefix := lua.CheckString(l, 3)
+		tableLog, err := client.fetchTableLog(repo, ref, prefix)
+		if err != nil {
+			lua.Errorf(l, err.Error())
+			panic("failed fetching table log")
+		}
+		luautil.DeepPush(l, tableLog)
+		return 1
+	}
+}
+
+func fetchFormattedChanges(client *DeltaClient) lua.Function {
+	return func(l *lua.State) int {
+		return 1
+	}
+}
+
+var functions = map[string]func(client *DeltaClient) lua.Function{
+	"get_table":               getTable,
+	"fetch_formatted_changes": fetchFormattedChanges,
+}
+
+type AccessProvider interface {
+	GetAccessProperties() (interface{}, error)
+}
+
+type AWSInfo struct {
+	AWSProps deltaStore.AWSProperties
+}
+
+type AWSInfo2 struct {
+	Region     string
+	DisableSsl bool
+	Key        string
+	Secret     string
+	Endpoint   string
+}
+
+func (awsI AWSInfo) GetAccessProperties() (interface{}, error) {
+	return awsI, nil
+}
+
+func newDelta(ctx context.Context) lua.Function {
+	// Factory method to create storage specific Delta Lake client
+	return func(l *lua.State) int {
+		var client *DeltaClient
+		st := lua.CheckString(l, 1)
+		switch storageType(st) {
+		case s3StorageType:
+			client = newS3DeltaClient(l, ctx)
+		default:
+			lua.Errorf(l, "unimplemented storage type")
+			panic("unimplemented storage type")
+		}
+		l.NewTable()
+		for name, goFn := range functions {
+			// -1: tbl
+			l.PushGoFunction(goFn(client))
+			// -1: fn, -2:tbl
+			l.SetField(-2, name)
+		}
+		return 1
+	}
+}
+
+func newS3DeltaClient(l *lua.State, ctx context.Context) *DeltaClient {
+	aki := lua.CheckString(l, 2)
+	sak := lua.CheckString(l, 3)
+	e := lua.CheckString(l, 4)
+	r := lua.OptString(l, 5, "us-east-1")
+	disableSsl := lua.CheckInteger(l, 6) == 1
+	awsProps := deltaStore.AWSProperties{
+		Region:         r,
+		ForcePathStyle: true,
+		DisableHTTPs:   disableSsl,
+		CredsProvider: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) {
+			return aws.Credentials{
+				AccessKeyID:     aki,
+				SecretAccessKey: sak,
+			}, nil
+		}),
+		Endpoint: e,
+	}
+	deltaStore.RegisterS3CompatBucketURLOpener("lakefs", &awsProps)
+
+	return &DeltaClient{accessProvider: AWSInfo{AWSProps: awsProps}, ctx: ctx}
+}
diff --git a/pkg/actions/lua/formats/formats.go b/pkg/actions/lua/formats/formats.go
new file mode 100644
index 00000000000..c89503350cd
--- /dev/null
+++ b/pkg/actions/lua/formats/formats.go
@@ -0,0 +1,17 @@
+package formats
+
+import (
+	"context"
+	"github.com/Shopify/go-lua"
+)
+
+func Open(l *lua.State, ctx context.Context) {
+	open := func(l *lua.State) int {
+		lua.NewLibrary(l, []lua.RegistryFunction{
+			{Name: "delta_client", Function: newDelta(ctx)},
+		})
+		return 1
+	}
+	lua.Require(l, "formats", open, false)
+	l.Pop(1)
+}
diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
new file mode 100644
index 00000000000..6434ea0d119
--- /dev/null
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -0,0 +1,49 @@
+local lakefs = require "lakefs"
+--local delta = require "delta"
+--local pathlib = require "path"
+--local json = require "encoding/json"
+local utils = require "lakefs/catalogexport/internal"
+
+local table_paths = args.table_paths
+local ns = args.storage_ns
+local repo = args.repository_id
+local commit_id = args.commit_id
+
+--[[
+ Changes list -> []SingleAction struct
+
+]]
+
+print(args)
+
+for _, path in ipairs(table_paths) do
+    local delta_log_path = path.join("/", path, "_delta_log")
+    local log_entry_objects = utils.lakefs_object_pager(lakefs, repo, commit_id, "", delta_log_path,"", 50)
+    for _, obj in ipairs(log_entry_objects) do
+        print(obj)
+    end
+    ---- lakefs.get_object(repo, commit_id, delta_log_path)
+    --local recent_changes = delta.fetch_formatted_changes(table, lakefs_client)
+    ----[[
+    --{
+    --    tableName: string
+    --    changes: [
+    --        content: string
+    --        entry_name: string
+    --    ]
+    --}
+    --]]
+    --if options.debug then
+    --    print("Recent changes:", recent_changes)
+    --end
+    --local recent_changes_json = json.unmarshal(recent_changes)
+    --local changes = {}
+    --for _, change in ipairs(recent_changes) do
+    --    local changeJson = json.marshal(change)
+    --    changes[changeJson.entry] = changeJson.changes
+    --end
+    --filesContents[]
+    --return {
+    --    table_input = table_input
+    --}
+end
diff --git a/pkg/actions/lua/lakefs/catalogexport/internal.lua b/pkg/actions/lua/lakefs/catalogexport/internal.lua
index 12c39562760..92fa4c79eec 100644
--- a/pkg/actions/lua/lakefs/catalogexport/internal.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/internal.lua
@@ -18,9 +18,9 @@ end
 
 local function short_digest(digest, len)
     return digest:sub(1, len or DEFAULT_SHORT_DIGEST_LEN)
-end 
+end
 
--- paginate lakefs api 
+-- paginate lakefs api
 local function lakefs_paginiated_api(api_call, after)
     local next_offset = after
     local has_more = true
@@ -38,7 +38,7 @@ local function lakefs_paginiated_api(api_call, after)
     end
 end
 
--- paginage over lakefs objects 
+-- paginage over lakefs objects
 local function lakefs_object_pager(lakefs_client, repo_id, commit_id, after, prefix, delimiter, page_size)
     return lakefs_paginiated_api(function(next_offset)
         return lakefs_client.list_objects(repo_id, commit_id, next_offset, prefix, delimiter, page_size or 30)
@@ -59,18 +59,34 @@ end
 
 local function parse_storage_uri(uri)
     local u = url.parse(uri)
-    return { 
+    return {
         protocol = u.scheme,
         bucket = u.host,
         key = (u.path:sub(0, 1) == "/") and u.path:sub(2) or u.path,
     }
 end
 
+local function sortedKeys(query, sortFunction)
+    local keys, len = {}, 0
+    for k,_ in pairs(query) do
+        len = len + 1
+        keys[len] = k
+    end
+    if sortFunction ~= nil then
+        table.sort(keys, sortFunction)
+    else
+        table.sort(keys)
+    end
+
+    return keys
+end
+
 return {
     deepcopy=deepcopy,
     parse_storage_uri=parse_storage_uri,
     short_digest=short_digest,
     ref_from_branch_or_tag=ref_from_branch_or_tag,
-    lakefs_object_pager=lakefs_object_pager, 
+    lakefs_object_pager=lakefs_object_pager,
     lakefs_paginiated_api=lakefs_paginiated_api,
-}
\ No newline at end of file
+    sortedKeys = sortedKeys
+}
diff --git a/pkg/actions/lua/lakefs/catalogexport/storage_utils.lua b/pkg/actions/lua/lakefs/catalogexport/storage_utils.lua
new file mode 100644
index 00000000000..d7f2b52b141
--- /dev/null
+++ b/pkg/actions/lua/lakefs/catalogexport/storage_utils.lua
@@ -0,0 +1,34 @@
+local utils = require("lakefs/catalogexport/internal")
+local pathlib = require("path")
+local url = require("net/url")
+
+--[[
+    ### Default exported table structure:
+
+    ${storageNamespace}
+    _lakefs/
+        exported/
+            ${ref}/
+                ${commitId}/
+                    ${tableName}/
+                        ...
+]]
+local function get_storage_uri_prefix(storage_ns, commit_id, action_info)
+    local branch_or_tag = utils.ref_from_branch_or_tag(action_info)
+    local sha = utils.short_digest(commit_id)
+    return pathlib.join("/", storage_ns, string.format("_lakefs/exported/%s/%s/", branch_or_tag, sha))
+end
+
+local function parse_storage_uri(uri)
+    local u = url.parse(uri)
+    return {
+        protocol = u.scheme,
+        bucket = u.host,
+        key = (u.path:sub(0, 1) == "/") and u.path:sub(2) or u.path,
+    }
+end
+
+return {
+    get_storage_uri_prefix = get_storage_uri_prefix,
+    parse_storage_uri = parse_storage_uri
+}

From 8e8ddcaf13c0cfe507431ab22e57c419d26567d6 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 23 Nov 2023 15:52:46 +0200
Subject: [PATCH 02/35] delta exporter

---
 .../lakefs/catalogexport/delta_exporter.lua   | 170 +++++++++++++-----
 1 file changed, 129 insertions(+), 41 deletions(-)

diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index 6434ea0d119..ae1b955190e 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -1,49 +1,137 @@
-local lakefs = require "lakefs"
---local delta = require "delta"
---local pathlib = require "path"
---local json = require "encoding/json"
-local utils = require "lakefs/catalogexport/internal"
-
-local table_paths = args.table_paths
-local ns = args.storage_ns
-local repo = args.repository_id
-local commit_id = args.commit_id
+local lakefs = require("lakefs")
+local formats = require("formats")
+local pathlib = require("path")
+local json = require("encoding/json")
+local storage_utils = require("lakefs/catalogexport/storage_utils")
+local utils = require("lakefs/catalogexport/internal")
 
 --[[
- Changes list -> []SingleAction struct
+    action_args:
+        - storage_namespace
+        - repo
+        - commit_id
+
+    export_delta_args:
+        - table_paths: ["path/to/table1", "path/to/table2", ...]
+        - lakefs_key
+        - lakefs_secret
+        - **might change** lakefs_server_url
+
+    storage_client:
+        - put_object: function(bucket, key, data)
 
 ]]
+local function export_delta_log(action, export_delta_args, storage_client)
+    local ns = action.storage_namespace
+    local repo = action.repository_id
+    local commit_id = action.commit_id
+    local table_paths = export_delta_args.table_paths
+    local lakefs_key = export_delta_args.lakefs_key
+    local lakefs_secret = export_delta_args.lakefs_secret
+    local lakefs_server_url = export_delta_args.lakefs_server_url
+    local disable_ssl = lakefs_server_url:find("http://", 1, true)
+
+    local delta = formats.delta_client("s3", lakefs_key, lakefs_secret, lakefs_server_url, "us-east-1", disable_ssl)
+
+    for _, path in ipairs(table_paths) do
+        print(path)
+        local t = delta.get_table(repo, commit_id, path)
+        local sortedKeys = utils.sortedKeys(t, nil)
+        --[[ Pairs of (version, map of json content):
+                (1,
+                {"commitInfo":{"timestamp":1699276565259,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"4","numOutputBytes":"1353"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.3.0","txnId":"eb6816ae-404f-4338-9e1a-2cb0a4626ab3"}}
+                {"add":{"path":"part-00000-a5a20e52-2b3d-440b-97a8-829fbc4a2804-c000.snappy.parquet","partitionValues":{},"size":1353,"modificationTime":1699276565000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"Hylak_id\":18,\"Lake_name\":\" \",\"Country\":\"Malawi\",\"Depth_m\":3.0},\"maxValues\":{\"Hylak_id\":16138,\"Lake_name\":\"Malombe\",\"Country\":\"Malawi\",\"Depth_m\":706.0},\"nullCount\":{\"Hylak_id\":0,\"Lake_name\":0,\"Country\":0,\"Depth_m\":0}}"}}
+                {"remove":{"path":"part-00000-d660b401-ceec-415a-a791-e8d1c7599e3d-c000.snappy.parquet","deletionTimestamp":1699276565259,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":82491}})
+        ]]
+        local table_log = {}
+        --local i = 0
+        local keyGenerator = generateKeyFunc()
+        for _, key in ipairs(sortedKeys) do
+            print(key)
+            local content = t[key]
+            local entry_log = {}
+            --[[
+                An array of entries:
+                    {"commitInfo":{"timestamp":1699276565259,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"4","numOutputBytes":"1353"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.3.0","txnId":"eb6816ae-404f-4338-9e1a-2cb0a4626ab3"}},
+                    {"add":{"path":"part-00000-a5a20e52-2b3d-440b-97a8-829fbc4a2804-c000.snappy.parquet","partitionValues":{},"size":1353,"modificationTime":1699276565000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"Hylak_id\":18,\"Lake_name\":\" \",\"Country\":\"Malawi\",\"Depth_m\":3.0},\"maxValues\":{\"Hylak_id\":16138,\"Lake_name\":\"Malombe\",\"Country\":\"Malawi\",\"Depth_m\":706.0},\"nullCount\":{\"Hylak_id\":0,\"Lake_name\":0,\"Country\":0,\"Depth_m\":0}}"}},
+                    {"remove":{"path":"part-00000-d660b401-ceec-415a-a791-e8d1c7599e3d-c000.snappy.parquet","deletionTimestamp":1699276565259,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":82491}}
+            ]]
+            for _, e in ipairs(content) do
+                print(e)
+                local entry = json.unmarshal(e)
+                local p = ""
+                if entry.add ~= nil then
+                    p = entry.add.path
+                elseif entry.remove ~= nil then
+                    p = entry.remove.path
+                end
+                if p ~= "" then
+                    local code, obj = lakefs.stat_object(repo, commit_id, pathlib.join("/",path, p))
+                    if code == 200 then
+                        local obj_stat = json.unmarshal(obj)
+                        local physical_path = obj_stat["physical_address"]
+                        if entry.add ~= nil then
+                            entry.add.path = physical_path
+                        elseif entry.remove ~= nil then
+                            entry.remove.path = physical_path
+                        end
+                    end
+                end
+                local entry_m = json.marshal(entry)
+                --print(entry_m)
+                table.insert(entry_log, entry_m)
+            end
+            table_log[string.format("%s.json", keyGenerator())] = entry_log
+            --i = i + 1
+        end
 
-print(args)
+        -- Get the table delta log physical location
+        local t_name = pathlib.parse(path)["base_name"]
+        local table_physical_path = pathlib.join("/", storage_utils.get_storage_uri_prefix(ns, commit_id, action), t_name, "_delta_log")
+        print(table_physical_path)
 
-for _, path in ipairs(table_paths) do
-    local delta_log_path = path.join("/", path, "_delta_log")
-    local log_entry_objects = utils.lakefs_object_pager(lakefs, repo, commit_id, "", delta_log_path,"", 50)
-    for _, obj in ipairs(log_entry_objects) do
-        print(obj)
+        -- Upload the log to this physical_address
+        local storage_props = storage_utils.parse_storage_uri(table_physical_path)
+        --[[
+            table_log:
+                {
+                    <version1>.json : [
+                        {"commitInfo":...}, {"add": ...}, {"remove": ...},...
+                    ],
+                    <version2>.json : [
+                        {"commitInfo":...}, {"add": ...}, {"remove": ...},...
+                    ],...
+                }
+        ]]
+        for entry_version, table_entry in pairs(table_log) do
+            local table_entry_string = ""
+            for _, content_entry in ipairs(table_entry) do
+                table_entry_string = table_entry_string .. content_entry
+            end
+            local version_key = storage_props.key .. "/" .. entry_version
+            print("Version key:\n" .. version_key .. "\n")
+            print("Version content:\n" .. table_entry_string .. "\n")
+            storage_client.put_object(storage_props.bucket, version_key, table_entry_string)
+        end
     end
-    ---- lakefs.get_object(repo, commit_id, delta_log_path)
-    --local recent_changes = delta.fetch_formatted_changes(table, lakefs_client)
-    ----[[
-    --{
-    --    tableName: string
-    --    changes: [
-    --        content: string
-    --        entry_name: string
-    --    ]
-    --}
-    --]]
-    --if options.debug then
-    --    print("Recent changes:", recent_changes)
-    --end
-    --local recent_changes_json = json.unmarshal(recent_changes)
-    --local changes = {}
-    --for _, change in ipairs(recent_changes) do
-    --    local changeJson = json.marshal(change)
-    --    changes[changeJson.entry] = changeJson.changes
-    --end
-    --filesContents[]
-    --return {
-    --    table_input = table_input
-    --}
 end
+
+function generateKeyFunc()
+    local current = 0
+    return function()
+        local delta_log_entry_length = 20
+        local key = tostring(current)
+        local padding_length = delta_log_entry_length - key:len()
+        local padded_key = ""
+        for _ = 1, padding_length do
+            padded_key = padded_key .. "0"
+        end
+        padded_key = padded_key .. key
+        current = current + 1
+        return padded_key
+    end
+end
+
+return {
+    export_delta_log = export_delta_log
+}

From 8bb3f1c06e0f98e291ad105f3d58029b406c4aba Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Mon, 27 Nov 2023 12:11:37 +0200
Subject: [PATCH 03/35] Use the earliest available version to build the Delta
 log

---
 pkg/actions/lua/formats/delta.go | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index 210128de619..ab1c5991c5a 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -44,15 +44,12 @@ func (dc *DeltaClient) buildS3DeltaTable(repo, ref, prefix string) (delta.Log, e
 	return delta.ForTableWithStore(url, config, &delta.SystemClock{}, &store)
 }
 func (dc *DeltaClient) buildLog(table delta.Log) (map[int64][]string, error) {
-	//s, err := table.Snapshot()
-	//if err != nil {
-	//	return nil, err
-	//}
-	//version := s.Version()
-
-	//versionLog, err := table.Changes(version, false)
-
-	versionLog, err := table.Changes(0, false)
+	s, err := table.Snapshot()
+	if err != nil {
+		return nil, err
+	}
+	version, _ := s.EarliestVersion()
+	versionLog, err := table.Changes(version, false)
 	if err != nil {
 		return nil, err
 	}

From 5adf15d61ded928049458c781fb6de7dceabf84d Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Mon, 27 Nov 2023 17:09:34 +0200
Subject: [PATCH 04/35] pass listen address to hooks

---
 cmd/lakefs/cmd/run.go              |  1 +
 pkg/actions/airflow.go             |  2 +-
 pkg/actions/hook.go                |  6 +--
 pkg/actions/lua.go                 | 24 +++++++-----
 pkg/actions/lua/formats/delta.go   | 61 +++++++++++++++---------------
 pkg/actions/lua/formats/formats.go |  4 +-
 pkg/actions/lua/open.go            |  3 +-
 pkg/actions/lua/stdlib.go          |  3 +-
 pkg/actions/service.go             | 44 +++++++++++----------
 pkg/actions/webhook.go             |  2 +-
 10 files changed, 81 insertions(+), 69 deletions(-)

diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go
index 68727e5f8ed..ecc6310a688 100644
--- a/cmd/lakefs/cmd/run.go
+++ b/cmd/lakefs/cmd/run.go
@@ -203,6 +203,7 @@ var runCmd = &cobra.Command{
 			idGen,
 			bufferedCollector,
 			actions.Config(cfg.Actions),
+			cfg.ListenAddress,
 		)
 
 		// wire actions into entry catalog
diff --git a/pkg/actions/airflow.go b/pkg/actions/airflow.go
index ebf3dbd0457..0f9555596d6 100644
--- a/pkg/actions/airflow.go
+++ b/pkg/actions/airflow.go
@@ -56,7 +56,7 @@ var (
 	errAirflowHookDAGFailed     = errors.New("airflow hook DAG failed")
 )
 
-func NewAirflowHook(h ActionHook, action *Action, cfg Config, endpoint *http.Server, collector stats.Collector) (Hook, error) {
+func NewAirflowHook(h ActionHook, action *Action, cfg Config, endpoint *http.Server, _ string, collector stats.Collector) (Hook, error) {
 	airflowHook := Airflow{
 		HookBase: HookBase{
 			ID:         h.ID,
diff --git a/pkg/actions/hook.go b/pkg/actions/hook.go
index ffb1bb806c6..a342488e818 100644
--- a/pkg/actions/hook.go
+++ b/pkg/actions/hook.go
@@ -24,7 +24,7 @@ type Hook interface {
 	Run(ctx context.Context, record graveler.HookRecord, buf *bytes.Buffer) error
 }
 
-type NewHookFunc func(ActionHook, *Action, Config, *http.Server, stats.Collector) (Hook, error)
+type NewHookFunc func(ActionHook, *Action, Config, *http.Server, string, stats.Collector) (Hook, error)
 
 type HookBase struct {
 	ID         string
@@ -41,10 +41,10 @@ var hooks = map[HookType]NewHookFunc{
 
 var ErrUnknownHookType = errors.New("unknown hook type")
 
-func NewHook(hook ActionHook, action *Action, cfg Config, server *http.Server, collector stats.Collector) (Hook, error) {
+func NewHook(hook ActionHook, action *Action, cfg Config, server *http.Server, listeningAddress string, collector stats.Collector) (Hook, error) {
 	f := hooks[hook.Type]
 	if f == nil {
 		return nil, fmt.Errorf("%w (%s)", ErrUnknownHookType, hook.Type)
 	}
-	return f(hook, action, cfg, server, collector)
+	return f(hook, action, cfg, server, listeningAddress, collector)
 }
diff --git a/pkg/actions/lua.go b/pkg/actions/lua.go
index 52de8790aa1..18a78681b62 100644
--- a/pkg/actions/lua.go
+++ b/pkg/actions/lua.go
@@ -25,10 +25,11 @@ import (
 
 type LuaHook struct {
 	HookBase
-	Script     string
-	ScriptPath string
-	Args       map[string]interface{}
-	collector  stats.Collector
+	Script           string
+	ScriptPath       string
+	Args             map[string]interface{}
+	collector        stats.Collector
+	listeningAddress string
 }
 
 func applyRecord(l *lua.State, actionName, hookID string, record graveler.HookRecord) {
@@ -88,7 +89,11 @@ func (h *LuaHook) Run(ctx context.Context, record graveler.HookRecord, buf *byte
 		return err
 	}
 	l := lua.NewState()
-	lualibs.OpenSafe(l, ctx, h.Config.Lua, &loggingBuffer{buf: buf, ctx: ctx})
+	osc := lualibs.OpenSafeConfig{
+		NetHTTPEnabled:   h.Config.Lua.NetHTTPEnabled,
+		ListeningAddress: h.listeningAddress,
+	}
+	lualibs.OpenSafe(l, ctx, osc, &loggingBuffer{buf: buf, ctx: ctx})
 	injectHookContext(l, ctx, user, h.Endpoint, h.Args)
 	applyRecord(l, h.ActionName, h.ID, record)
 
@@ -190,7 +195,7 @@ func DescendArgs(args interface{}) (interface{}, error) {
 	}
 }
 
-func NewLuaHook(h ActionHook, action *Action, cfg Config, e *http.Server, collector stats.Collector) (Hook, error) {
+func NewLuaHook(h ActionHook, action *Action, cfg Config, e *http.Server, listeningAddress string, collector stats.Collector) (Hook, error) {
 	// optional args
 	args := make(map[string]interface{})
 	argsVal, hasArgs := h.Properties["args"]
@@ -249,8 +254,9 @@ func NewLuaHook(h ActionHook, action *Action, cfg Config, e *http.Server, collec
 			Config:     cfg,
 			Endpoint:   e,
 		},
-		ScriptPath: scriptFile,
-		Args:       args,
-		collector:  collector,
+		ScriptPath:       scriptFile,
+		Args:             args,
+		collector:        collector,
+		listeningAddress: listeningAddress,
 	}, nil
 }
diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index ab1c5991c5a..04b97a754ce 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -7,9 +7,11 @@ import (
 	"github.com/Shopify/go-lua"
 	"github.com/aws/aws-sdk-go-v2/aws"
 	delta "github.com/csimplestring/delta-go"
+	"github.com/csimplestring/delta-go/storage"
 	deltaStore "github.com/csimplestring/delta-go/store"
 	luautil "github.com/treeverse/lakefs/pkg/actions/lua/util"
-	"path"
+	"net/url"
+	"strings"
 )
 
 type storageType string
@@ -21,27 +23,32 @@ const (
 )
 
 type DeltaClient struct {
-	accessProvider AccessProvider
-	ctx            context.Context
+	accessProvider   AccessProvider
+	ctx              context.Context
+	listeningAddress string
 }
 
-func (dc *DeltaClient) s3Fetcher(repo, ref, prefix string) (map[int64][]string, error) {
-	table, err := dc.buildS3DeltaTable(repo, ref, prefix)
+func (dc *DeltaClient) fetchS3Table(repo, ref, prefix string) (map[int64][]string, error) {
+	table, err := dc.getS3DeltaTable(repo, ref, prefix)
 	if err != nil {
 		return nil, err
 	}
 	return dc.buildLog(table)
 }
-func (dc *DeltaClient) buildS3DeltaTable(repo, ref, prefix string) (delta.Log, error) {
+func (dc *DeltaClient) getS3DeltaTable(repo, ref, prefix string) (delta.Log, error) {
 	awsInfo := dc.accessProvider.(AWSInfo)
 	config := delta.Config{StoreType: string(s3StorageType)}
-	s3LogStore, err := deltaStore.NewS3CompatLogStore(&awsInfo.AWSProps, repo, path.Join(ref, prefix))
+	u := fmt.Sprintf("lakefs://%s/%s/%s", repo, ref, prefix)
+	parsedURL, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+	s3LogStore, err := deltaStore.NewS3CompatLogStore(&awsInfo.AWSProps, parsedURL)
 	if err != nil {
 		return nil, err
 	}
 	store := deltaStore.Store(s3LogStore)
-	url := fmt.Sprintf("lakefs://%s/%s/%s", repo, ref, prefix)
-	return delta.ForTableWithStore(url, config, &delta.SystemClock{}, &store)
+	return delta.ForTableWithStore(u, config, &delta.SystemClock{}, &store)
 }
 func (dc *DeltaClient) buildLog(table delta.Log) (map[int64][]string, error) {
 	s, err := table.Snapshot()
@@ -75,7 +82,7 @@ func (dc *DeltaClient) fetchTableLog(repo, ref, prefix string) (map[int64][]stri
 	ap, _ := dc.accessProvider.GetAccessProperties()
 	switch ap.(type) {
 	case AWSInfo:
-		return dc.s3Fetcher(repo, ref, prefix)
+		return dc.fetchS3Table(repo, ref, prefix)
 	default:
 		return nil, errors.New("unimplemented provider")
 	}
@@ -113,29 +120,26 @@ type AccessProvider interface {
 }
 
 type AWSInfo struct {
-	AWSProps deltaStore.AWSProperties
-}
-
-type AWSInfo2 struct {
-	Region     string
-	DisableSsl bool
-	Key        string
-	Secret     string
-	Endpoint   string
+	AWSProps storage.AWSProperties
 }
 
 func (awsI AWSInfo) GetAccessProperties() (interface{}, error) {
 	return awsI, nil
 }
 
-func newDelta(ctx context.Context) lua.Function {
+func newDelta(ctx context.Context, listenAddress string) lua.Function {
+	if strings.HasPrefix(listenAddress, ":") {
+		// workaround in case we listen on all interfaces without specifying ip
+		listenAddress = fmt.Sprintf("localhost%s", listenAddress)
+	}
+	listenAddress = fmt.Sprintf("http://%s", listenAddress)
 	// Factory method to create storage specific Delta Lake client
 	return func(l *lua.State) int {
 		var client *DeltaClient
 		st := lua.CheckString(l, 1)
 		switch storageType(st) {
 		case s3StorageType:
-			client = newS3DeltaClient(l, ctx)
+			client = newS3DeltaClient(l, ctx, listenAddress)
 		default:
 			lua.Errorf(l, "unimplemented storage type")
 			panic("unimplemented storage type")
@@ -151,25 +155,22 @@ func newDelta(ctx context.Context) lua.Function {
 	}
 }
 
-func newS3DeltaClient(l *lua.State, ctx context.Context) *DeltaClient {
+func newS3DeltaClient(l *lua.State, ctx context.Context, listenAddress string) *DeltaClient {
 	aki := lua.CheckString(l, 2)
 	sak := lua.CheckString(l, 3)
-	e := lua.CheckString(l, 4)
-	r := lua.OptString(l, 5, "us-east-1")
-	disableSsl := lua.CheckInteger(l, 6) == 1
-	awsProps := deltaStore.AWSProperties{
+	r := lua.OptString(l, 4, "us-east-1")
+	awsProps := storage.AWSProperties{
 		Region:         r,
 		ForcePathStyle: true,
-		DisableHTTPs:   disableSsl,
 		CredsProvider: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) {
 			return aws.Credentials{
 				AccessKeyID:     aki,
 				SecretAccessKey: sak,
 			}, nil
 		}),
-		Endpoint: e,
+		Endpoint: listenAddress,
 	}
-	deltaStore.RegisterS3CompatBucketURLOpener("lakefs", &awsProps)
+	storage.RegisterS3CompatBucketURLOpener("lakefs", &awsProps)
 
-	return &DeltaClient{accessProvider: AWSInfo{AWSProps: awsProps}, ctx: ctx}
+	return &DeltaClient{accessProvider: AWSInfo{AWSProps: awsProps}, ctx: ctx, listeningAddress: listenAddress}
 }
diff --git a/pkg/actions/lua/formats/formats.go b/pkg/actions/lua/formats/formats.go
index c89503350cd..4f6a4cab767 100644
--- a/pkg/actions/lua/formats/formats.go
+++ b/pkg/actions/lua/formats/formats.go
@@ -5,10 +5,10 @@ import (
 	"github.com/Shopify/go-lua"
 )
 
-func Open(l *lua.State, ctx context.Context) {
+func Open(l *lua.State, ctx context.Context, listeningAddress string) {
 	open := func(l *lua.State) int {
 		lua.NewLibrary(l, []lua.RegistryFunction{
-			{Name: "delta_client", Function: newDelta(ctx)},
+			{Name: "delta_client", Function: newDelta(ctx, listeningAddress)},
 		})
 		return 1
 	}
diff --git a/pkg/actions/lua/open.go b/pkg/actions/lua/open.go
index 83301da177b..04c26e61a6c 100644
--- a/pkg/actions/lua/open.go
+++ b/pkg/actions/lua/open.go
@@ -2,7 +2,6 @@ package lua
 
 import (
 	"context"
-
 	"github.com/Shopify/go-lua"
 	"github.com/treeverse/lakefs/pkg/actions/lua/crypto/aes"
 	"github.com/treeverse/lakefs/pkg/actions/lua/crypto/hmac"
@@ -12,6 +11,7 @@ import (
 	"github.com/treeverse/lakefs/pkg/actions/lua/encoding/json"
 	"github.com/treeverse/lakefs/pkg/actions/lua/encoding/parquet"
 	"github.com/treeverse/lakefs/pkg/actions/lua/encoding/yaml"
+	"github.com/treeverse/lakefs/pkg/actions/lua/formats"
 	"github.com/treeverse/lakefs/pkg/actions/lua/net/http"
 	"github.com/treeverse/lakefs/pkg/actions/lua/net/url"
 	"github.com/treeverse/lakefs/pkg/actions/lua/path"
@@ -45,6 +45,7 @@ func Open(l *lua.State, ctx context.Context, cfg OpenSafeConfig) {
 	aws.Open(l, ctx)
 	gcloud.Open(l, ctx)
 	url.Open(l)
+	formats.Open(l, ctx, cfg.ListeningAddress)
 	if cfg.NetHTTPEnabled {
 		http.Open(l)
 	}
diff --git a/pkg/actions/lua/stdlib.go b/pkg/actions/lua/stdlib.go
index a04242ce9dc..dce88106a86 100644
--- a/pkg/actions/lua/stdlib.go
+++ b/pkg/actions/lua/stdlib.go
@@ -349,7 +349,8 @@ func BaseOpen(buf io.StringWriter) glua.Function {
 }
 
 type OpenSafeConfig struct {
-	NetHTTPEnabled bool
+	NetHTTPEnabled   bool
+	ListeningAddress string
 }
 
 func OpenSafe(l *glua.State, ctx context.Context, cfg OpenSafeConfig, buf io.StringWriter) {
diff --git a/pkg/actions/service.go b/pkg/actions/service.go
index ffb3fc5fade..136bcfdf4c0 100644
--- a/pkg/actions/service.go
+++ b/pkg/actions/service.go
@@ -48,16 +48,17 @@ type Config struct {
 // the run data to the blockstore and to the actions.Store (which is a
 // fancy name for a DB - kv style or postgres directly)
 type StoreService struct {
-	Store    Store
-	idGen    IDGenerator
-	Source   Source
-	Writer   OutputWriter
-	ctx      context.Context
-	cancel   context.CancelFunc
-	wg       sync.WaitGroup
-	stats    stats.Collector
-	cfg      Config
-	endpoint *http.Server
+	Store            Store
+	idGen            IDGenerator
+	Source           Source
+	Writer           OutputWriter
+	ctx              context.Context
+	cancel           context.CancelFunc
+	wg               sync.WaitGroup
+	stats            stats.Collector
+	cfg              Config
+	endpoint         *http.Server
+	listeningAddress string
 }
 
 type Task struct {
@@ -213,18 +214,19 @@ type Service interface {
 	graveler.HooksHandler
 }
 
-func NewService(ctx context.Context, store Store, source Source, writer OutputWriter, idGen IDGenerator, stats stats.Collector, cfg Config) *StoreService {
+func NewService(ctx context.Context, store Store, source Source, writer OutputWriter, idGen IDGenerator, stats stats.Collector, cfg Config, listeningAddress string) *StoreService {
 	ctx, cancel := context.WithCancel(ctx)
 	return &StoreService{
-		Store:  store,
-		Source: source,
-		Writer: writer,
-		ctx:    ctx,
-		idGen:  idGen,
-		cancel: cancel,
-		wg:     sync.WaitGroup{},
-		stats:  stats,
-		cfg:    cfg,
+		Store:            store,
+		Source:           source,
+		Writer:           writer,
+		ctx:              ctx,
+		idGen:            idGen,
+		cancel:           cancel,
+		wg:               sync.WaitGroup{},
+		stats:            stats,
+		cfg:              cfg,
+		listeningAddress: listeningAddress,
 	}
 }
 
@@ -310,7 +312,7 @@ func (s *StoreService) allocateTasks(runID string, actions []*Action) ([][]*Task
 	for actionIdx, action := range actions {
 		var actionTasks []*Task
 		for hookIdx, hook := range action.Hooks {
-			h, err := NewHook(hook, action, s.cfg, s.endpoint, s.stats)
+			h, err := NewHook(hook, action, s.cfg, s.endpoint, s.listeningAddress, s.stats)
 			if err != nil {
 				return nil, err
 			}
diff --git a/pkg/actions/webhook.go b/pkg/actions/webhook.go
index 3aa1ec577b6..e1cc462620b 100644
--- a/pkg/actions/webhook.go
+++ b/pkg/actions/webhook.go
@@ -36,7 +36,7 @@ var (
 	errWebhookWrongFormat   = errors.New("webhook wrong format")
 )
 
-func NewWebhook(h ActionHook, action *Action, cfg Config, e *http.Server, collector stats.Collector) (Hook, error) {
+func NewWebhook(h ActionHook, action *Action, cfg Config, e *http.Server, _ string, collector stats.Collector) (Hook, error) {
 	url, ok := h.Properties[webhookURLPropertyKey]
 	if !ok {
 		return nil, fmt.Errorf("missing url: %w", errWebhookWrongFormat)

From 37adb940c66185e0868e1f08c972ce73f37d00d2 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Mon, 27 Nov 2023 17:19:10 +0200
Subject: [PATCH 05/35] add stat_object api call to the lakefs lua client

---
 pkg/actions/lua/lakefs/client.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/pkg/actions/lua/lakefs/client.go b/pkg/actions/lua/lakefs/client.go
index acc0df3334c..ea432d47f1b 100644
--- a/pkg/actions/lua/lakefs/client.go
+++ b/pkg/actions/lua/lakefs/client.go
@@ -183,6 +183,27 @@ func OpenClient(l *lua.State, ctx context.Context, user *model.User, server *htt
 				l.PushString(rr.Body.String())
 				return 2
 			}},
+			{Name: "stat_object", Function: func(state *lua.State) int {
+				repo := lua.CheckString(l, 1)
+				ref := lua.CheckString(l, 2)
+				reqURL, err := url.JoinPath("/repositories", repo, "refs", ref, "objects", "stat")
+				if err != nil {
+					check(l, err)
+				}
+				req, err := newLakeFSJSONRequest(ctx, user, http.MethodGet, reqURL, nil)
+				if err != nil {
+					check(l, err)
+				}
+				// query params
+				q := req.URL.Query()
+				q.Add("path", lua.CheckString(l, 3))
+				req.URL.RawQuery = q.Encode()
+				rr := httptest.NewRecorder()
+				server.Handler.ServeHTTP(rr, req)
+				l.PushInteger(rr.Code)
+				l.PushString(rr.Body.String())
+				return 2
+			}},
 			{Name: "diff_branch", Function: func(state *lua.State) int {
 				repo := lua.CheckString(l, 1)
 				branch := lua.CheckString(l, 2)

From cd7b8b66e4d6df53e4c91f0e0a16c40e4633168c Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Mon, 27 Nov 2023 18:33:47 +0200
Subject: [PATCH 06/35] add get_repo to eventually fetch its storage namespace

---
 pkg/actions/lua/lakefs/client.go | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pkg/actions/lua/lakefs/client.go b/pkg/actions/lua/lakefs/client.go
index ea432d47f1b..d84399d180b 100644
--- a/pkg/actions/lua/lakefs/client.go
+++ b/pkg/actions/lua/lakefs/client.go
@@ -204,6 +204,22 @@ func OpenClient(l *lua.State, ctx context.Context, user *model.User, server *htt
 				l.PushString(rr.Body.String())
 				return 2
 			}},
+			{Name: "get_repo", Function: func(state *lua.State) int {
+				repo := lua.CheckString(l, 1)
+				reqURL, err := url.JoinPath("/repositories", repo)
+				req, err := newLakeFSJSONRequest(ctx, user, http.MethodGet, reqURL, nil)
+				if err != nil {
+					check(l, err)
+				}
+				// query params
+				q := req.URL.Query()
+				req.URL.RawQuery = q.Encode()
+				rr := httptest.NewRecorder()
+				server.Handler.ServeHTTP(rr, req)
+				l.PushInteger(rr.Code)
+				l.PushString(rr.Body.String())
+				return 2
+			}},
 			{Name: "diff_branch", Function: func(state *lua.State) int {
 				repo := lua.CheckString(l, 1)
 				branch := lua.CheckString(l, 2)

From c3bc9824bda93cf8c247d955404fd41384cc8759 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 16:16:03 +0200
Subject: [PATCH 07/35] delta lake exporter example

---
 examples/hooks/delta_lake_S3_export.lua | 30 +++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 examples/hooks/delta_lake_S3_export.lua

diff --git a/examples/hooks/delta_lake_S3_export.lua b/examples/hooks/delta_lake_S3_export.lua
new file mode 100644
index 00000000000..e9e414f7508
--- /dev/null
+++ b/examples/hooks/delta_lake_S3_export.lua
@@ -0,0 +1,30 @@
+--[[
+    action_args:
+        - repo
+        - commit_id
+
+    export_delta_args:
+        - table_paths: ["path/to/table1", "path/to/table2", ...]
+        - lakefs_key
+        - lakefs_secret
+        - region
+
+    storage_client:
+        - put_object: function(bucket, key, data)
+]]
+
+local export_delta_args = {
+    table_paths = args.table_paths,
+    lakefs_key = args.lakefs.access_key_id,
+    lakefs_secret = args.lakefs.secret_access_key,
+    region = args.aws.aws_region
+}
+
+local aws = require("aws")
+local delta = require("lakefs/catalogexport/delta_exporter")
+local sc = aws.s3_client(args.aws.aws_access_key_id, args.aws.aws_secret_access_key, args.aws.aws_region)
+
+local delta_table_locations = delta.export_delta_log(action, export_delta_args, sc)
+for t, loc in pairs(delta_table_locations) do
+    print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n")
+end

From 8a81e4e240a77bae41c89d939df9c20e710061bd Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 16:16:58 +0200
Subject: [PATCH 08/35] add error return. remove redundant lines. add godocs
 for some funcitons

---
 pkg/actions/lua/formats/delta.go | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index 04b97a754ce..1550e7e8e70 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -17,9 +17,7 @@ import (
 type storageType string
 
 const (
-	s3StorageType    storageType = "s3"
-	gcsStorageType   storageType = "gcs"
-	azureStorageType storageType = "azure"
+	s3StorageType storageType = "s3"
 )
 
 type DeltaClient struct {
@@ -55,7 +53,10 @@ func (dc *DeltaClient) buildLog(table delta.Log) (map[int64][]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	version, _ := s.EarliestVersion()
+	version, err := s.EarliestVersion()
+	if err != nil {
+		return nil, err
+	}
 	versionLog, err := table.Changes(version, false)
 	if err != nil {
 		return nil, err
@@ -104,17 +105,11 @@ func getTable(client *DeltaClient) lua.Function {
 	}
 }
 
-func fetchFormattedChanges(client *DeltaClient) lua.Function {
-	return func(l *lua.State) int {
-		return 1
-	}
-}
-
 var functions = map[string]func(client *DeltaClient) lua.Function{
-	"get_table":               getTable,
-	"fetch_formatted_changes": fetchFormattedChanges,
+	"get_table": getTable,
 }
 
+// AccessProvider is used to provide different expected access properties to different storage providers
 type AccessProvider interface {
 	GetAccessProperties() (interface{}, error)
 }
@@ -127,13 +122,13 @@ func (awsI AWSInfo) GetAccessProperties() (interface{}, error) {
 	return awsI, nil
 }
 
+// newDelta is a factory function to create server/cloud specific Delta Lake client
 func newDelta(ctx context.Context, listenAddress string) lua.Function {
 	if strings.HasPrefix(listenAddress, ":") {
 		// workaround in case we listen on all interfaces without specifying ip
 		listenAddress = fmt.Sprintf("localhost%s", listenAddress)
 	}
 	listenAddress = fmt.Sprintf("http://%s", listenAddress)
-	// Factory method to create storage specific Delta Lake client
 	return func(l *lua.State) int {
 		var client *DeltaClient
 		st := lua.CheckString(l, 1)
@@ -146,9 +141,7 @@ func newDelta(ctx context.Context, listenAddress string) lua.Function {
 		}
 		l.NewTable()
 		for name, goFn := range functions {
-			// -1: tbl
 			l.PushGoFunction(goFn(client))
-			// -1: fn, -2:tbl
 			l.SetField(-2, name)
 		}
 		return 1

From c4f520afc832364ebf2d1add04df1cf7e8d4d9fa Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 16:18:36 +0200
Subject: [PATCH 09/35] extract get_storage_namespace to the internal lua file.
 change response of delta_exporter, remove prints, remove old impl

---
 .../lakefs/catalogexport/delta_exporter.lua   | 50 +++++++++----------
 .../lua/lakefs/catalogexport/internal.lua     | 12 ++++-
 2 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index ae1b955190e..72b848d4bd4 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -6,35 +6,35 @@ local storage_utils = require("lakefs/catalogexport/storage_utils")
 local utils = require("lakefs/catalogexport/internal")
 
 --[[
-    action_args:
-        - storage_namespace
-        - repo
+    action:
+        - repository_id
         - commit_id
 
-    export_delta_args:
+    hook_args:
         - table_paths: ["path/to/table1", "path/to/table2", ...]
         - lakefs_key
         - lakefs_secret
-        - **might change** lakefs_server_url
+        - region
 
     storage_client:
         - put_object: function(bucket, key, data)
 
 ]]
-local function export_delta_log(action, export_delta_args, storage_client)
-    local ns = action.storage_namespace
+local function export_delta_log(action, hook_args, storage_client)
     local repo = action.repository_id
     local commit_id = action.commit_id
-    local table_paths = export_delta_args.table_paths
-    local lakefs_key = export_delta_args.lakefs_key
-    local lakefs_secret = export_delta_args.lakefs_secret
-    local lakefs_server_url = export_delta_args.lakefs_server_url
-    local disable_ssl = lakefs_server_url:find("http://", 1, true)
-
-    local delta = formats.delta_client("s3", lakefs_key, lakefs_secret, lakefs_server_url, "us-east-1", disable_ssl)
+    local table_paths = hook_args.table_paths
+    local lakefs_key = hook_args.lakefs_key
+    local lakefs_secret = hook_args.lakefs_secret
+    local region = hook_args.region
 
+    local ns = utils.get_storage_namespace(repo)
+    if ns == nil then
+        error("failed getting storage namespace for repo " .. repo)
+    end
+    local delta = formats.delta_client("s3", lakefs_key, lakefs_secret, region)
+    local response = {}
     for _, path in ipairs(table_paths) do
-        print(path)
         local t = delta.get_table(repo, commit_id, path)
         local sortedKeys = utils.sortedKeys(t, nil)
         --[[ Pairs of (version, map of json content):
@@ -44,10 +44,8 @@ local function export_delta_log(action, export_delta_args, storage_client)
                 {"remove":{"path":"part-00000-d660b401-ceec-415a-a791-e8d1c7599e3d-c000.snappy.parquet","deletionTimestamp":1699276565259,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":82491}})
         ]]
         local table_log = {}
-        --local i = 0
-        local keyGenerator = generateKeyFunc()
+        local keyGenerator = delta_log_entry_key_generator()
         for _, key in ipairs(sortedKeys) do
-            print(key)
             local content = t[key]
             local entry_log = {}
             --[[
@@ -57,7 +55,6 @@ local function export_delta_log(action, export_delta_args, storage_client)
                     {"remove":{"path":"part-00000-d660b401-ceec-415a-a791-e8d1c7599e3d-c000.snappy.parquet","deletionTimestamp":1699276565259,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":82491}}
             ]]
             for _, e in ipairs(content) do
-                print(e)
                 local entry = json.unmarshal(e)
                 local p = ""
                 if entry.add ~= nil then
@@ -78,20 +75,19 @@ local function export_delta_log(action, export_delta_args, storage_client)
                     end
                 end
                 local entry_m = json.marshal(entry)
-                --print(entry_m)
                 table.insert(entry_log, entry_m)
             end
             table_log[string.format("%s.json", keyGenerator())] = entry_log
-            --i = i + 1
         end
 
         -- Get the table delta log physical location
         local t_name = pathlib.parse(path)["base_name"]
-        local table_physical_path = pathlib.join("/", storage_utils.get_storage_uri_prefix(ns, commit_id, action), t_name, "_delta_log")
-        print(table_physical_path)
+        local table_export_prefix = storage_utils.get_storage_uri_prefix(ns, commit_id, action)
+        local table_physical_path = pathlib.join("/", table_export_prefix, t_name)
+        local table_log_physical_path = pathlib.join("/", table_physical_path, "_delta_log")
 
         -- Upload the log to this physical_address
-        local storage_props = storage_utils.parse_storage_uri(table_physical_path)
+        local storage_props = storage_utils.parse_storage_uri(table_log_physical_path)
         --[[
             table_log:
                 {
@@ -109,14 +105,14 @@ local function export_delta_log(action, export_delta_args, storage_client)
                 table_entry_string = table_entry_string .. content_entry
             end
             local version_key = storage_props.key .. "/" .. entry_version
-            print("Version key:\n" .. version_key .. "\n")
-            print("Version content:\n" .. table_entry_string .. "\n")
             storage_client.put_object(storage_props.bucket, version_key, table_entry_string)
         end
+        response[t_name] = table_physical_path
     end
+    return response
 end
 
-function generateKeyFunc()
+function delta_log_entry_key_generator()
     local current = 0
     return function()
         local delta_log_entry_length = 20
diff --git a/pkg/actions/lua/lakefs/catalogexport/internal.lua b/pkg/actions/lua/lakefs/catalogexport/internal.lua
index 92fa4c79eec..99b4ac518b5 100644
--- a/pkg/actions/lua/lakefs/catalogexport/internal.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/internal.lua
@@ -81,6 +81,15 @@ local function sortedKeys(query, sortFunction)
     return keys
 end
 
+local function get_storage_namespace(repo)
+    local code, obj = lakefs.get_repo(repo)
+    if code == 200 then
+        local obj_repo = json.unmarshal(obj)
+        return obj_repo["storage_namespace"]
+    end
+    return nil
+end
+
 return {
     deepcopy=deepcopy,
     parse_storage_uri=parse_storage_uri,
@@ -88,5 +97,6 @@ return {
     ref_from_branch_or_tag=ref_from_branch_or_tag,
     lakefs_object_pager=lakefs_object_pager,
     lakefs_paginiated_api=lakefs_paginiated_api,
-    sortedKeys = sortedKeys
+    sortedKeys = sortedKeys,
+    get_storage_namespace = get_storage_namespace,
 }

From b2b81aab6484b394d79dd055e65017aa842d62cc Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 16:22:44 +0200
Subject: [PATCH 10/35] remove comment

---
 pkg/actions/lua/encoding/json/json.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pkg/actions/lua/encoding/json/json.go b/pkg/actions/lua/encoding/json/json.go
index d8973eac493..37542000136 100644
--- a/pkg/actions/lua/encoding/json/json.go
+++ b/pkg/actions/lua/encoding/json/json.go
@@ -54,7 +54,6 @@ func jsonMarshal(l *lua.State) int {
 			e.SetIndent(pre, ind)
 		}
 	}
-	//e.SetIndent("", "  ")
 	err = e.Encode(t)
 	check(l, err)
 	l.PushString(buf.String())

From 587ed18411b81495dac9882f880ca18ab67237f4 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 16:53:46 +0200
Subject: [PATCH 11/35] go mod tidy

---
 go.mod |  6 +++---
 go.sum | 45 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/go.mod b/go.mod
index 97183e2155f..cb4ac46c613 100644
--- a/go.mod
+++ b/go.mod
@@ -82,6 +82,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.23.2
 	github.com/aws/smithy-go v1.15.0
 	github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc
+	github.com/csimplestring/delta-go v0.0.0-20231105162402-9b93ca02cedf
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/georgysavva/scany/v2 v2.0.0
 	github.com/go-co-op/gocron v1.35.2
@@ -123,11 +124,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
 	github.com/barweiss/go-tuple v1.1.1 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
-	github.com/csimplestring/delta-go v0.0.0-20231105162402-9b93ca02cedf // indirect
 	github.com/deckarep/golang-set/v2 v2.3.1 // indirect
 	github.com/fraugster/parquet-go v0.12.0 // indirect
 	github.com/getsentry/sentry-go v0.16.0 // indirect
-	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/glog v1.1.0 // indirect
 	github.com/google/flatbuffers v2.0.0+incompatible // indirect
@@ -139,7 +138,6 @@ require (
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lib/pq v1.10.9 // indirect
 	github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/oklog/run v1.0.0 // indirect
@@ -248,3 +246,5 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
+
+replace github.com/csimplestring/delta-go => github.com/treeverse/delta-go v0.0.0-20231128165607-87acc8332e05
diff --git a/go.sum b/go.sum
index d9a3d814095..78dfdb0dd36 100644
--- a/go.sum
+++ b/go.sum
@@ -116,6 +116,7 @@ github.com/Azure/go-amqp v0.16.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fw
 github.com/Azure/go-amqp v0.16.4/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
@@ -128,6 +129,7 @@ github.com/Azure/go-autorest/autorest/azure/auth v0.5.9/go.mod h1:hg3/1yw0Bq87O3
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=
 github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
 github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
@@ -164,6 +166,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/ahmetb/go-linq/v3 v3.2.0 h1:BEuMfp+b59io8g5wYzNoFe9pWPalRklhlhbiU3hYZDE=
+github.com/ahmetb/go-linq/v3 v3.2.0/go.mod h1:haQ3JfOeWK8HpVxMtHHEMPVgBKiYyQ+f1/kLZh/cj9U=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs=
@@ -175,8 +179,10 @@ github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CK
 github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk=
 github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
@@ -257,6 +263,8 @@ github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnw
 github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
 github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
+github.com/barweiss/go-tuple v1.1.1 h1:0dLwRqJHHpU9c5iJLF9Xjl5yEPDXwh5FhigydBoM5MM=
+github.com/barweiss/go-tuple v1.1.1/go.mod h1:oX3AtZyuGnCuK3B4PlUIfR9spxyJ42bgTSqCI9GkPB0=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -332,6 +340,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A=
+github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/deepmap/oapi-codegen v1.5.6 h1:hlUtA13SL2HNoC/5vXDFdGm2AaN1j9/rUu7zedjBiqg=
 github.com/deepmap/oapi-codegen v1.5.6/go.mod h1:NoliSkZp7cRAkisw65+PUtvgy56f21QQesX1tVUzS8g=
 github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU=
@@ -385,6 +395,8 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fraugster/parquet-go v0.12.0 h1:1slnC5y2VWEOUSlzbeXatM0BvSWcLUDsR/EcZsXXCZc=
+github.com/fraugster/parquet-go v0.12.0/go.mod h1:dGzUxdNqXsAijatByVgbAWVPlFirnhknQbdazcUIjY0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
@@ -439,8 +451,8 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
-github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
 github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
@@ -539,8 +551,11 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE=
 github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk=
 github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks=
+github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
+github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
@@ -578,6 +593,7 @@ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
 github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
 github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
 github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
@@ -815,6 +831,7 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
@@ -840,6 +857,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg=
 github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
+github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -916,6 +935,9 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa
 github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY=
 github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg=
+github.com/repeale/fp-go v0.11.1 h1:Q/e+gNyyHaxKAyfdbBqvip3DxhVWH453R+kthvSr9Mk=
+github.com/repeale/fp-go v0.11.1/go.mod h1:4KrwQJB1VRY+06CA+jTc4baZetr6o2PeuqnKr5ybQUc=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -928,6 +950,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rotisserie/eris v0.5.4 h1:Il6IvLdAapsMhvuOahHWiBnl1G++Q0/L5UIkI5mARSk=
+github.com/rotisserie/eris v0.5.4/go.mod h1:Z/kgYTJiJtocxCbFfvRmO+QejApzG6zpyky9G1A4g9s=
 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -944,15 +968,20 @@ github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9c
 github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/samber/mo v1.10.0 h1:dxwUGzd5i2FyPQJZzTIdexUuQC6WhXlq7zUJ9o7XCqI=
+github.com/samber/mo v1.10.0/go.mod h1:BfkrCPuYzVG3ZljnZB783WIJIGk1mcZr9c9CPf8tAxs=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
 github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -1008,12 +1037,16 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/thanhpk/randstr v1.0.6 h1:psAOktJFD4vV9NEVb3qkhRSMvYh4ORRaj1+w/hn4B+o=
 github.com/thanhpk/randstr v1.0.6/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U=
+github.com/treeverse/delta-go v0.0.0-20231128165607-87acc8332e05 h1:2J9ZsK33cYZRANUjEeYYqPPlv3kA9ETTnbUyqCd/IYQ=
+github.com/treeverse/delta-go v0.0.0-20231128165607-87acc8332e05/go.mod h1:KUVLx8Ya8UkX9BZSx2MehNINLdFYyLuvsaNXWU1k208=
 github.com/tsenart/vegeta/v12 v12.11.1 h1:Rbwe7Zxr7sJ+BDTReemeQalYPvKiSV+O7nwmUs20B3E=
 github.com/tsenart/vegeta/v12 v12.11.1/go.mod h1:swiFmrgpqj2llHURgHYFRFN0tfrIrlnspg01HjwOnSQ=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA=
+github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -1031,6 +1064,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
+github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww=
 github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM=
 github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA=
@@ -1075,8 +1110,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
-go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
-go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/ratelimit v0.3.0 h1:IdZd9wqvFXnvLvSEBo0KPcGfkoBGNkpTHlrE3Rcjkjw=
 go.uber.org/ratelimit v0.3.0/go.mod h1:So5LG7CV1zWpY1sHe+DXTJqQvOx+FFPFaAs2SnoyBaI=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
@@ -1085,6 +1120,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg=
+gocloud.dev v0.34.0 h1:LzlQY+4l2cMtuNfwT2ht4+fiXwWf/NmPTnXUlLmGif4=
+gocloud.dev v0.34.0/go.mod h1:psKOachbnvY3DAOPbsFVmLIErwsbWPUG2H5i65D38vE=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

From bde6581e01c23706c61b8bf2e63856bac3b41678 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 17:09:11 +0200
Subject: [PATCH 12/35] pass aws properties directly to fetchS3Table

---
 pkg/actions/lua/formats/delta.go | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index 1550e7e8e70..e3a1baf44c9 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -26,22 +26,21 @@ type DeltaClient struct {
 	listeningAddress string
 }
 
-func (dc *DeltaClient) fetchS3Table(repo, ref, prefix string) (map[int64][]string, error) {
-	table, err := dc.getS3DeltaTable(repo, ref, prefix)
+func (dc *DeltaClient) fetchS3Table(repo, ref, prefix string, awsProps *storage.AWSProperties) (map[int64][]string, error) {
+	table, err := dc.getS3DeltaTable(repo, ref, prefix, awsProps)
 	if err != nil {
 		return nil, err
 	}
 	return dc.buildLog(table)
 }
-func (dc *DeltaClient) getS3DeltaTable(repo, ref, prefix string) (delta.Log, error) {
-	awsInfo := dc.accessProvider.(AWSInfo)
+func (dc *DeltaClient) getS3DeltaTable(repo, ref, prefix string, awsProps *storage.AWSProperties) (delta.Log, error) {
 	config := delta.Config{StoreType: string(s3StorageType)}
 	u := fmt.Sprintf("lakefs://%s/%s/%s", repo, ref, prefix)
 	parsedURL, err := url.Parse(u)
 	if err != nil {
 		return nil, err
 	}
-	s3LogStore, err := deltaStore.NewS3CompatLogStore(&awsInfo.AWSProps, parsedURL)
+	s3LogStore, err := deltaStore.NewS3CompatLogStore(awsProps, parsedURL)
 	if err != nil {
 		return nil, err
 	}
@@ -81,9 +80,9 @@ func (dc *DeltaClient) buildLog(table delta.Log) (map[int64][]string, error) {
 
 func (dc *DeltaClient) fetchTableLog(repo, ref, prefix string) (map[int64][]string, error) {
 	ap, _ := dc.accessProvider.GetAccessProperties()
-	switch ap.(type) {
+	switch access := ap.(type) {
 	case AWSInfo:
-		return dc.fetchS3Table(repo, ref, prefix)
+		return dc.fetchS3Table(repo, ref, prefix, &access.AWSProps)
 	default:
 		return nil, errors.New("unimplemented provider")
 	}

From 4df6b5a8d5b3366b9aa33abc62a202eb33474148 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 18:31:50 +0200
Subject: [PATCH 13/35] json encoding refactor

---
 pkg/actions/lua/encoding/json/json.go | 30 +++++++--------------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/pkg/actions/lua/encoding/json/json.go b/pkg/actions/lua/encoding/json/json.go
index 37542000136..a95ac7f7f9d 100644
--- a/pkg/actions/lua/encoding/json/json.go
+++ b/pkg/actions/lua/encoding/json/json.go
@@ -50,9 +50,8 @@ func jsonMarshal(l *lua.State) int {
 		rot = vot.Interface().(map[string]any)
 	}
 	if rot != nil {
-		if pre, ind, ok := fetchIndentProps(rot); ok {
-			e.SetIndent(pre, ind)
-		}
+		pre, ind := fetchIndentProps(rot)
+		e.SetIndent(pre, ind)
 	}
 	err = e.Encode(t)
 	check(l, err)
@@ -60,29 +59,16 @@ func jsonMarshal(l *lua.State) int {
 	return 1
 }
 
-func fetchIndentProps(rot map[string]any) (string, string, bool) {
-	var prefix *string
-	var indent *string
+func fetchIndentProps(rot map[string]any) (string, string) {
+	prefix := ""
+	indent := ""
 	if p, ok := rot["prefix"]; ok {
-		sp := p.(string)
-		prefix = &sp
+		prefix = p.(string)
 	}
 	if i, ok := rot["indent"]; ok {
-		si := i.(string)
-		indent = &si
+		indent = i.(string)
 	}
-	if prefix != nil || indent != nil {
-		if prefix == nil {
-			defaultPref := ""
-			prefix = &defaultPref
-		}
-		if indent == nil {
-			defaultIndent := ""
-			indent = &defaultIndent
-		}
-		return *prefix, *indent, true
-	}
-	return "", "", false
+	return prefix, indent
 }
 
 func jsonUnmarshal(l *lua.State) int {

From c49c42e4f71fbc5ecb6b2cb9555ccc135dad2a0a Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 19:00:31 +0200
Subject: [PATCH 14/35] fix load test

---
 pkg/loadtest/local_load_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/loadtest/local_load_test.go b/pkg/loadtest/local_load_test.go
index b366cff0707..14e5a721a7e 100644
--- a/pkg/loadtest/local_load_test.go
+++ b/pkg/loadtest/local_load_test.go
@@ -69,7 +69,7 @@ func TestLocalLoad(t *testing.T) {
 	outputWriter := catalog.NewActionsOutputWriter(c.BlockAdapter)
 
 	// wire actions
-	actionsService := actions.NewService(ctx, actions.NewActionsKVStore(kvStore), source, outputWriter, &actions.DecreasingIDGenerator{}, &stats.NullCollector{}, actions.Config{Enabled: true})
+	actionsService := actions.NewService(ctx, actions.NewActionsKVStore(kvStore), source, outputWriter, &actions.DecreasingIDGenerator{}, &stats.NullCollector{}, actions.Config{Enabled: true}, "")
 	c.SetHooksHandler(actionsService)
 
 	credentials, err := setup.CreateAdminUser(ctx, authService, conf, superuser)

From e5013b91ee93a547372d16e12cb0972015e506d9 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 19:27:20 +0200
Subject: [PATCH 15/35] fix tests

---
 pkg/actions/kv_run_results_iterator_test.go | 2 +-
 pkg/actions/service_test.go                 | 2 +-
 pkg/api/serve_test.go                       | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pkg/actions/kv_run_results_iterator_test.go b/pkg/actions/kv_run_results_iterator_test.go
index 620505ada7d..3798cdfaebf 100644
--- a/pkg/actions/kv_run_results_iterator_test.go
+++ b/pkg/actions/kv_run_results_iterator_test.go
@@ -138,7 +138,7 @@ func createTestData(t *testing.T, ctx context.Context, kvStore kv.Store) (map[st
 	writer := mock.NewMockOutputWriter(ctrl)
 	mockStatsCollector := NewActionStatsMockCollector()
 	testSource := mock.NewMockSource(ctrl)
-	actionService := actions.NewService(ctx, actions.NewActionsKVStore(kvStore), testSource, writer, &TestDecreasingIDGenerator{num: math.MaxInt32}, &mockStatsCollector, actions.Config{})
+	actionService := actions.NewService(ctx, actions.NewActionsKVStore(kvStore), testSource, writer, &TestDecreasingIDGenerator{num: math.MaxInt32}, &mockStatsCollector, actions.Config{}, "")
 	msgIdx := 0
 	run := actions.RunResultData{
 		RunId:     "",
diff --git a/pkg/actions/service_test.go b/pkg/actions/service_test.go
index c89f2d554a8..8f0c2585dc9 100644
--- a/pkg/actions/service_test.go
+++ b/pkg/actions/service_test.go
@@ -63,7 +63,7 @@ func GetKVService(t *testing.T, ctx context.Context, source actions.Source, writ
 	kvStore := kvtest.GetStore(ctx, t)
 	cfg := actions.Config{Enabled: runHooks}
 	cfg.Lua.NetHTTPEnabled = true
-	return actions.NewService(ctx, actions.NewActionsKVStore(kvStore), source, writer, &actions.DecreasingIDGenerator{}, stats, cfg)
+	return actions.NewService(ctx, actions.NewActionsKVStore(kvStore), source, writer, &actions.DecreasingIDGenerator{}, stats, cfg, "")
 }
 
 func TestServiceRun(t *testing.T) {
diff --git a/pkg/api/serve_test.go b/pkg/api/serve_test.go
index 4c6300a282d..78b1ef6732c 100644
--- a/pkg/api/serve_test.go
+++ b/pkg/api/serve_test.go
@@ -171,6 +171,7 @@ func setupHandlerWithWalkerFactory(t testing.TB, factory catalog.WalkerFactory)
 		idGen,
 		collector,
 		actionsConfig,
+		"",
 	)
 
 	c.SetHooksHandler(actionsService)

From 7318c64d9ca4813a7f2caea0d106e69a1e9fdab0 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 19:35:58 +0200
Subject: [PATCH 16/35] fix tests

---
 pkg/actions/lua_test.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/actions/lua_test.go b/pkg/actions/lua_test.go
index 48ac17c6ed9..ad71c068a16 100644
--- a/pkg/actions/lua_test.go
+++ b/pkg/actions/lua_test.go
@@ -46,7 +46,7 @@ func TestNewLuaHook(t *testing.T) {
 				NetHTTPEnabled: true,
 			},
 		},
-		nil, &mockStatsCollector)
+		nil, "", &mockStatsCollector)
 	if err != nil {
 		t.Errorf("unexpedcted error: %v", err)
 	}
@@ -77,7 +77,7 @@ func TestLuaRun(t *testing.T) {
 				NetHTTPEnabled: true,
 			},
 		},
-		nil, &mockStatsCollector)
+		nil, "", &mockStatsCollector)
 	if err != nil {
 		t.Errorf("unexpedcted error: %v", err)
 	}
@@ -130,7 +130,7 @@ func TestLuaRun_NetHttpDisabled(t *testing.T) {
 			Hooks:       nil,
 		},
 		actions.Config{Enabled: true},
-		nil, &mockStatsCollector)
+		nil, "", &mockStatsCollector)
 	if err != nil {
 		t.Errorf("unexpedcted error: %v", err)
 	}
@@ -281,7 +281,7 @@ print(code .. " " .. body .. " " .. status)
 						NetHTTPEnabled: true,
 					},
 				},
-				nil, &mockStatsCollector)
+				nil, "", &mockStatsCollector)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
@@ -400,7 +400,7 @@ func TestLuaRunTable(t *testing.T) {
 						NetHTTPEnabled: true,
 					},
 				},
-				nil, &mockStatsCollector)
+				nil, "", &mockStatsCollector)
 			if err != nil {
 				t.Errorf("unexpedcted error: %v", err)
 			}

From 2bac82e8521fcc43bed6e651444a1677640b0582 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 29 Nov 2023 19:36:48 +0200
Subject: [PATCH 17/35] goimports

---
 pkg/actions/lua/formats/delta.go   | 5 +++--
 pkg/actions/lua/formats/formats.go | 1 +
 pkg/actions/lua/open.go            | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index e3a1baf44c9..802e75ef14a 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -4,14 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/url"
+	"strings"
+
 	"github.com/Shopify/go-lua"
 	"github.com/aws/aws-sdk-go-v2/aws"
 	delta "github.com/csimplestring/delta-go"
 	"github.com/csimplestring/delta-go/storage"
 	deltaStore "github.com/csimplestring/delta-go/store"
 	luautil "github.com/treeverse/lakefs/pkg/actions/lua/util"
-	"net/url"
-	"strings"
 )
 
 type storageType string
diff --git a/pkg/actions/lua/formats/formats.go b/pkg/actions/lua/formats/formats.go
index 4f6a4cab767..8a4a1146ad6 100644
--- a/pkg/actions/lua/formats/formats.go
+++ b/pkg/actions/lua/formats/formats.go
@@ -2,6 +2,7 @@ package formats
 
 import (
 	"context"
+
 	"github.com/Shopify/go-lua"
 )
 
diff --git a/pkg/actions/lua/open.go b/pkg/actions/lua/open.go
index 04c26e61a6c..df5d3361afa 100644
--- a/pkg/actions/lua/open.go
+++ b/pkg/actions/lua/open.go
@@ -2,6 +2,7 @@ package lua
 
 import (
 	"context"
+
 	"github.com/Shopify/go-lua"
 	"github.com/treeverse/lakefs/pkg/actions/lua/crypto/aes"
 	"github.com/treeverse/lakefs/pkg/actions/lua/crypto/hmac"

From 9bd7a76991cdaeebc8af60b88fb78091151ba2f5 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 11:55:02 +0200
Subject: [PATCH 18/35] add indentation to lua test

---
 pkg/actions/testdata/lua/json_marshal_action.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/actions/testdata/lua/json_marshal_action.lua b/pkg/actions/testdata/lua/json_marshal_action.lua
index 9ca43d32eb0..22c2b988882 100644
--- a/pkg/actions/testdata/lua/json_marshal_action.lua
+++ b/pkg/actions/testdata/lua/json_marshal_action.lua
@@ -1,3 +1,3 @@
 json = require("encoding/json")
 
-print(json.marshal(action))
\ No newline at end of file
+print(json.marshal(action, "  "))

From 855a4432080422f23a6a1125cd1526b58a6010ba Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 12:01:16 +0200
Subject: [PATCH 19/35] remove storage_utils.lua

---
 .../lakefs/catalogexport/delta_exporter.lua   | 14 ++++----
 .../lakefs/catalogexport/storage_utils.lua    | 34 -------------------
 2 files changed, 8 insertions(+), 40 deletions(-)
 delete mode 100644 pkg/actions/lua/lakefs/catalogexport/storage_utils.lua

diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index 72b848d4bd4..5d7bbd890d5 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -2,7 +2,6 @@ local lakefs = require("lakefs")
 local formats = require("formats")
 local pathlib = require("path")
 local json = require("encoding/json")
-local storage_utils = require("lakefs/catalogexport/storage_utils")
 local utils = require("lakefs/catalogexport/internal")
 
 --[[
@@ -39,9 +38,12 @@ local function export_delta_log(action, hook_args, storage_client)
         local sortedKeys = utils.sortedKeys(t, nil)
         --[[ Pairs of (version, map of json content):
                 (1,
-                {"commitInfo":{"timestamp":1699276565259,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"4","numOutputBytes":"1353"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.3.0","txnId":"eb6816ae-404f-4338-9e1a-2cb0a4626ab3"}}
-                {"add":{"path":"part-00000-a5a20e52-2b3d-440b-97a8-829fbc4a2804-c000.snappy.parquet","partitionValues":{},"size":1353,"modificationTime":1699276565000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"Hylak_id\":18,\"Lake_name\":\" \",\"Country\":\"Malawi\",\"Depth_m\":3.0},\"maxValues\":{\"Hylak_id\":16138,\"Lake_name\":\"Malombe\",\"Country\":\"Malawi\",\"Depth_m\":706.0},\"nullCount\":{\"Hylak_id\":0,\"Lake_name\":0,\"Country\":0,\"Depth_m\":0}}"}}
-                {"remove":{"path":"part-00000-d660b401-ceec-415a-a791-e8d1c7599e3d-c000.snappy.parquet","deletionTimestamp":1699276565259,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":82491}})
+                {
+                    {"commitInfo":{"timestamp":1699276565259,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"4","numOutputBytes":"1353"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.3.0","txnId":"eb6816ae-404f-4338-9e1a-2cb0a4626ab3"}}
+                    {"add":{"path":"part-00000-a5a20e52-2b3d-440b-97a8-829fbc4a2804-c000.snappy.parquet","partitionValues":{},"size":1353,"modificationTime":1699276565000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"Hylak_id\":18,\"Lake_name\":\" \",\"Country\":\"Malawi\",\"Depth_m\":3.0},\"maxValues\":{\"Hylak_id\":16138,\"Lake_name\":\"Malombe\",\"Country\":\"Malawi\",\"Depth_m\":706.0},\"nullCount\":{\"Hylak_id\":0,\"Lake_name\":0,\"Country\":0,\"Depth_m\":0}}"}}
+                    {"remove":{"path":"part-00000-d660b401-ceec-415a-a791-e8d1c7599e3d-c000.snappy.parquet","deletionTimestamp":1699276565259,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":82491}}
+                }
+                )
         ]]
         local table_log = {}
         local keyGenerator = delta_log_entry_key_generator()
@@ -82,12 +84,12 @@ local function export_delta_log(action, hook_args, storage_client)
 
         -- Get the table delta log physical location
         local t_name = pathlib.parse(path)["base_name"]
-        local table_export_prefix = storage_utils.get_storage_uri_prefix(ns, commit_id, action)
+        local table_export_prefix = utils.get_storage_uri_prefix(ns, commit_id, action)
         local table_physical_path = pathlib.join("/", table_export_prefix, t_name)
         local table_log_physical_path = pathlib.join("/", table_physical_path, "_delta_log")
 
         -- Upload the log to this physical_address
-        local storage_props = storage_utils.parse_storage_uri(table_log_physical_path)
+        local storage_props = utils.parse_storage_uri(table_log_physical_path)
         --[[
             table_log:
                 {
diff --git a/pkg/actions/lua/lakefs/catalogexport/storage_utils.lua b/pkg/actions/lua/lakefs/catalogexport/storage_utils.lua
deleted file mode 100644
index d7f2b52b141..00000000000
--- a/pkg/actions/lua/lakefs/catalogexport/storage_utils.lua
+++ /dev/null
@@ -1,34 +0,0 @@
-local utils = require("lakefs/catalogexport/internal")
-local pathlib = require("path")
-local url = require("net/url")
-
---[[
-    ### Default exported table structure:
-
-    ${storageNamespace}
-    _lakefs/
-        exported/
-            ${ref}/
-                ${commitId}/
-                    ${tableName}/
-                        ...
-]]
-local function get_storage_uri_prefix(storage_ns, commit_id, action_info)
-    local branch_or_tag = utils.ref_from_branch_or_tag(action_info)
-    local sha = utils.short_digest(commit_id)
-    return pathlib.join("/", storage_ns, string.format("_lakefs/exported/%s/%s/", branch_or_tag, sha))
-end
-
-local function parse_storage_uri(uri)
-    local u = url.parse(uri)
-    return {
-        protocol = u.scheme,
-        bucket = u.host,
-        key = (u.path:sub(0, 1) == "/") and u.path:sub(2) or u.path,
-    }
-end
-
-return {
-    get_storage_uri_prefix = get_storage_uri_prefix,
-    parse_storage_uri = parse_storage_uri
-}

From 74eb76b54b61d7f1b118349f3c4e99fd6be9134e Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 12:30:22 +0200
Subject: [PATCH 20/35] Rename `ListeningAddress`  (and `listeningAddress`) to
 `ServerAddress`. Lua refactoring Remove unneeded `listeningAddress` member
 from `DeltaClient`

---
 pkg/actions/hook.go                           |  4 +-
 pkg/actions/lua.go                            | 24 +++++-----
 pkg/actions/lua/formats/delta.go              | 31 ++++++-------
 pkg/actions/lua/formats/formats.go            |  4 +-
 .../lakefs/catalogexport/delta_exporter.lua   | 34 +++++++-------
 .../lua/lakefs/catalogexport/internal.lua     |  6 +--
 pkg/actions/lua/open.go                       |  2 +-
 pkg/actions/lua/stdlib.go                     |  4 +-
 pkg/actions/service.go                        | 46 +++++++++----------
 9 files changed, 75 insertions(+), 80 deletions(-)

diff --git a/pkg/actions/hook.go b/pkg/actions/hook.go
index a342488e818..fe6dd77fa65 100644
--- a/pkg/actions/hook.go
+++ b/pkg/actions/hook.go
@@ -41,10 +41,10 @@ var hooks = map[HookType]NewHookFunc{
 
 var ErrUnknownHookType = errors.New("unknown hook type")
 
-func NewHook(hook ActionHook, action *Action, cfg Config, server *http.Server, listeningAddress string, collector stats.Collector) (Hook, error) {
+func NewHook(hook ActionHook, action *Action, cfg Config, server *http.Server, serverAddress string, collector stats.Collector) (Hook, error) {
 	f := hooks[hook.Type]
 	if f == nil {
 		return nil, fmt.Errorf("%w (%s)", ErrUnknownHookType, hook.Type)
 	}
-	return f(hook, action, cfg, server, listeningAddress, collector)
+	return f(hook, action, cfg, server, serverAddress, collector)
 }
diff --git a/pkg/actions/lua.go b/pkg/actions/lua.go
index 18a78681b62..913c457bba7 100644
--- a/pkg/actions/lua.go
+++ b/pkg/actions/lua.go
@@ -25,11 +25,11 @@ import (
 
 type LuaHook struct {
 	HookBase
-	Script           string
-	ScriptPath       string
-	Args             map[string]interface{}
-	collector        stats.Collector
-	listeningAddress string
+	Script        string
+	ScriptPath    string
+	Args          map[string]interface{}
+	collector     stats.Collector
+	serverAddress string
 }
 
 func applyRecord(l *lua.State, actionName, hookID string, record graveler.HookRecord) {
@@ -90,8 +90,8 @@ func (h *LuaHook) Run(ctx context.Context, record graveler.HookRecord, buf *byte
 	}
 	l := lua.NewState()
 	osc := lualibs.OpenSafeConfig{
-		NetHTTPEnabled:   h.Config.Lua.NetHTTPEnabled,
-		ListeningAddress: h.listeningAddress,
+		NetHTTPEnabled: h.Config.Lua.NetHTTPEnabled,
+		ServerAddress:  h.serverAddress,
 	}
 	lualibs.OpenSafe(l, ctx, osc, &loggingBuffer{buf: buf, ctx: ctx})
 	injectHookContext(l, ctx, user, h.Endpoint, h.Args)
@@ -195,7 +195,7 @@ func DescendArgs(args interface{}) (interface{}, error) {
 	}
 }
 
-func NewLuaHook(h ActionHook, action *Action, cfg Config, e *http.Server, listeningAddress string, collector stats.Collector) (Hook, error) {
+func NewLuaHook(h ActionHook, action *Action, cfg Config, e *http.Server, serverAddress string, collector stats.Collector) (Hook, error) {
 	// optional args
 	args := make(map[string]interface{})
 	argsVal, hasArgs := h.Properties["args"]
@@ -254,9 +254,9 @@ func NewLuaHook(h ActionHook, action *Action, cfg Config, e *http.Server, listen
 			Config:     cfg,
 			Endpoint:   e,
 		},
-		ScriptPath:       scriptFile,
-		Args:             args,
-		collector:        collector,
-		listeningAddress: listeningAddress,
+		ScriptPath:    scriptFile,
+		Args:          args,
+		collector:     collector,
+		serverAddress: serverAddress,
 	}, nil
 }
diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index 802e75ef14a..b194673c203 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -22,9 +22,8 @@ const (
 )
 
 type DeltaClient struct {
-	accessProvider   AccessProvider
-	ctx              context.Context
-	listeningAddress string
+	accessProvider AccessProvider
+	ctx            context.Context
 }
 
 func (dc *DeltaClient) fetchS3Table(repo, ref, prefix string, awsProps *storage.AWSProperties) (map[int64][]string, error) {
@@ -123,18 +122,18 @@ func (awsI AWSInfo) GetAccessProperties() (interface{}, error) {
 }
 
 // newDelta is a factory function to create server/cloud specific Delta Lake client
-func newDelta(ctx context.Context, listenAddress string) lua.Function {
-	if strings.HasPrefix(listenAddress, ":") {
+func newDelta(ctx context.Context, serverAddress string) lua.Function {
+	if strings.HasPrefix(serverAddress, ":") {
 		// workaround in case we listen on all interfaces without specifying ip
-		listenAddress = fmt.Sprintf("localhost%s", listenAddress)
+		serverAddress = fmt.Sprintf("localhost%s", serverAddress)
 	}
-	listenAddress = fmt.Sprintf("http://%s", listenAddress)
+	serverAddress = fmt.Sprintf("http://%s", serverAddress)
 	return func(l *lua.State) int {
 		var client *DeltaClient
 		st := lua.CheckString(l, 1)
 		switch storageType(st) {
 		case s3StorageType:
-			client = newS3DeltaClient(l, ctx, listenAddress)
+			client = newS3DeltaClient(l, ctx, serverAddress)
 		default:
 			lua.Errorf(l, "unimplemented storage type")
 			panic("unimplemented storage type")
@@ -148,22 +147,22 @@ func newDelta(ctx context.Context, listenAddress string) lua.Function {
 	}
 }
 
-func newS3DeltaClient(l *lua.State, ctx context.Context, listenAddress string) *DeltaClient {
-	aki := lua.CheckString(l, 2)
-	sak := lua.CheckString(l, 3)
-	r := lua.OptString(l, 4, "us-east-1")
+func newS3DeltaClient(l *lua.State, ctx context.Context, serverAddress string) *DeltaClient {
+	accessKeyID := lua.CheckString(l, 2)
+	secretAccessKey := lua.CheckString(l, 3)
+	r := lua.CheckString(l, 4)
 	awsProps := storage.AWSProperties{
 		Region:         r,
 		ForcePathStyle: true,
 		CredsProvider: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) {
 			return aws.Credentials{
-				AccessKeyID:     aki,
-				SecretAccessKey: sak,
+				AccessKeyID:     accessKeyID,
+				SecretAccessKey: secretAccessKey,
 			}, nil
 		}),
-		Endpoint: listenAddress,
+		Endpoint: serverAddress,
 	}
 	storage.RegisterS3CompatBucketURLOpener("lakefs", &awsProps)
 
-	return &DeltaClient{accessProvider: AWSInfo{AWSProps: awsProps}, ctx: ctx, listeningAddress: listenAddress}
+	return &DeltaClient{accessProvider: AWSInfo{AWSProps: awsProps}, ctx: ctx}
 }
diff --git a/pkg/actions/lua/formats/formats.go b/pkg/actions/lua/formats/formats.go
index 8a4a1146ad6..ab9bcaf9605 100644
--- a/pkg/actions/lua/formats/formats.go
+++ b/pkg/actions/lua/formats/formats.go
@@ -6,10 +6,10 @@ import (
 	"github.com/Shopify/go-lua"
 )
 
-func Open(l *lua.State, ctx context.Context, listeningAddress string) {
+func Open(l *lua.State, ctx context.Context, serverAddress string) {
 	open := func(l *lua.State) int {
 		lua.NewLibrary(l, []lua.RegistryFunction{
-			{Name: "delta_client", Function: newDelta(ctx, listeningAddress)},
+			{Name: "delta_client", Function: newDelta(ctx, serverAddress)},
 		})
 		return 1
 	}
diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index 5d7bbd890d5..d144ff60507 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -4,6 +4,22 @@ local pathlib = require("path")
 local json = require("encoding/json")
 local utils = require("lakefs/catalogexport/internal")
 
+local function delta_log_entry_key_generator()
+    local current = 0
+    return function()
+        local delta_log_entry_length = 20
+        local key = tostring(current)
+        local padding_length = delta_log_entry_length - key:len()
+        local padded_key = ""
+        for _ = 1, padding_length do
+            padded_key = padded_key .. "0"
+        end
+        padded_key = padded_key .. key
+        current = current + 1
+        return padded_key
+    end
+end
+
 --[[
     action:
         - repository_id
@@ -35,7 +51,7 @@ local function export_delta_log(action, hook_args, storage_client)
     local response = {}
     for _, path in ipairs(table_paths) do
         local t = delta.get_table(repo, commit_id, path)
-        local sortedKeys = utils.sortedKeys(t, nil)
+        local sortedKeys = utils.sortedKeys(t)
         --[[ Pairs of (version, map of json content):
                 (1,
                 {
@@ -114,22 +130,6 @@ local function export_delta_log(action, hook_args, storage_client)
     return response
 end
 
-function delta_log_entry_key_generator()
-    local current = 0
-    return function()
-        local delta_log_entry_length = 20
-        local key = tostring(current)
-        local padding_length = delta_log_entry_length - key:len()
-        local padded_key = ""
-        for _ = 1, padding_length do
-            padded_key = padded_key .. "0"
-        end
-        padded_key = padded_key .. key
-        current = current + 1
-        return padded_key
-    end
-end
-
 return {
     export_delta_log = export_delta_log
 }
diff --git a/pkg/actions/lua/lakefs/catalogexport/internal.lua b/pkg/actions/lua/lakefs/catalogexport/internal.lua
index 99b4ac518b5..d9bb4bf607d 100644
--- a/pkg/actions/lua/lakefs/catalogexport/internal.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/internal.lua
@@ -72,11 +72,7 @@ local function sortedKeys(query, sortFunction)
         len = len + 1
         keys[len] = k
     end
-    if sortFunction ~= nil then
-        table.sort(keys, sortFunction)
-    else
-        table.sort(keys)
-    end
+    table.sort(keys, sortFunction)
 
     return keys
 end
diff --git a/pkg/actions/lua/open.go b/pkg/actions/lua/open.go
index df5d3361afa..c0eef50ddaa 100644
--- a/pkg/actions/lua/open.go
+++ b/pkg/actions/lua/open.go
@@ -46,7 +46,7 @@ func Open(l *lua.State, ctx context.Context, cfg OpenSafeConfig) {
 	aws.Open(l, ctx)
 	gcloud.Open(l, ctx)
 	url.Open(l)
-	formats.Open(l, ctx, cfg.ListeningAddress)
+	formats.Open(l, ctx, cfg.ServerAddress)
 	if cfg.NetHTTPEnabled {
 		http.Open(l)
 	}
diff --git a/pkg/actions/lua/stdlib.go b/pkg/actions/lua/stdlib.go
index dce88106a86..f420fa48f3d 100644
--- a/pkg/actions/lua/stdlib.go
+++ b/pkg/actions/lua/stdlib.go
@@ -349,8 +349,8 @@ func BaseOpen(buf io.StringWriter) glua.Function {
 }
 
 type OpenSafeConfig struct {
-	NetHTTPEnabled   bool
-	ListeningAddress string
+	NetHTTPEnabled bool
+	ServerAddress  string
 }
 
 func OpenSafe(l *glua.State, ctx context.Context, cfg OpenSafeConfig, buf io.StringWriter) {
diff --git a/pkg/actions/service.go b/pkg/actions/service.go
index 136bcfdf4c0..1ff8f33a213 100644
--- a/pkg/actions/service.go
+++ b/pkg/actions/service.go
@@ -48,17 +48,17 @@ type Config struct {
 // the run data to the blockstore and to the actions.Store (which is a
 // fancy name for a DB - kv style or postgres directly)
 type StoreService struct {
-	Store            Store
-	idGen            IDGenerator
-	Source           Source
-	Writer           OutputWriter
-	ctx              context.Context
-	cancel           context.CancelFunc
-	wg               sync.WaitGroup
-	stats            stats.Collector
-	cfg              Config
-	endpoint         *http.Server
-	listeningAddress string
+	Store         Store
+	idGen         IDGenerator
+	Source        Source
+	Writer        OutputWriter
+	ctx           context.Context
+	cancel        context.CancelFunc
+	wg            sync.WaitGroup
+	stats         stats.Collector
+	cfg           Config
+	endpoint      *http.Server
+	serverAddress string
 }
 
 type Task struct {
@@ -214,19 +214,19 @@ type Service interface {
 	graveler.HooksHandler
 }
 
-func NewService(ctx context.Context, store Store, source Source, writer OutputWriter, idGen IDGenerator, stats stats.Collector, cfg Config, listeningAddress string) *StoreService {
+func NewService(ctx context.Context, store Store, source Source, writer OutputWriter, idGen IDGenerator, stats stats.Collector, cfg Config, serverAddress string) *StoreService {
 	ctx, cancel := context.WithCancel(ctx)
 	return &StoreService{
-		Store:            store,
-		Source:           source,
-		Writer:           writer,
-		ctx:              ctx,
-		idGen:            idGen,
-		cancel:           cancel,
-		wg:               sync.WaitGroup{},
-		stats:            stats,
-		cfg:              cfg,
-		listeningAddress: listeningAddress,
+		Store:         store,
+		Source:        source,
+		Writer:        writer,
+		ctx:           ctx,
+		idGen:         idGen,
+		cancel:        cancel,
+		wg:            sync.WaitGroup{},
+		stats:         stats,
+		cfg:           cfg,
+		serverAddress: serverAddress,
 	}
 }
 
@@ -312,7 +312,7 @@ func (s *StoreService) allocateTasks(runID string, actions []*Action) ([][]*Task
 	for actionIdx, action := range actions {
 		var actionTasks []*Task
 		for hookIdx, hook := range action.Hooks {
-			h, err := NewHook(hook, action, s.cfg, s.endpoint, s.listeningAddress, s.stats)
+			h, err := NewHook(hook, action, s.cfg, s.endpoint, s.serverAddress, s.stats)
 			if err != nil {
 				return nil, err
 			}

From 0866a3c6c8476ae59ea4480440a0913bc61f0679 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 12:51:47 +0200
Subject: [PATCH 21/35] fix test

---
 pkg/actions/testdata/lua/json_marshal_action.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/actions/testdata/lua/json_marshal_action.lua b/pkg/actions/testdata/lua/json_marshal_action.lua
index 22c2b988882..25ef469990a 100644
--- a/pkg/actions/testdata/lua/json_marshal_action.lua
+++ b/pkg/actions/testdata/lua/json_marshal_action.lua
@@ -1,3 +1,3 @@
 json = require("encoding/json")
 
-print(json.marshal(action, "  "))
+print(json.marshal(action, {indent = "  "}))

From d0bb96e188d37715bbf8400a388d34dbd83ad3aa Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 13:59:39 +0200
Subject: [PATCH 22/35] lint

---
 pkg/actions/lua/formats/delta.go | 5 +++--
 pkg/actions/lua/lakefs/client.go | 3 +++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index b194673c203..b82b4f171a7 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -21,6 +21,8 @@ const (
 	s3StorageType storageType = "s3"
 )
 
+var errUnimplementedProvided = errors.New("unimplemented provider")
+
 type DeltaClient struct {
 	accessProvider AccessProvider
 	ctx            context.Context
@@ -84,9 +86,8 @@ func (dc *DeltaClient) fetchTableLog(repo, ref, prefix string) (map[int64][]stri
 	case AWSInfo:
 		return dc.fetchS3Table(repo, ref, prefix, &access.AWSProps)
 	default:
-		return nil, errors.New("unimplemented provider")
+		return nil, errUnimplementedProvided
 	}
-
 }
 
 func getTable(client *DeltaClient) lua.Function {
diff --git a/pkg/actions/lua/lakefs/client.go b/pkg/actions/lua/lakefs/client.go
index d84399d180b..cae9d3e5186 100644
--- a/pkg/actions/lua/lakefs/client.go
+++ b/pkg/actions/lua/lakefs/client.go
@@ -207,6 +207,9 @@ func OpenClient(l *lua.State, ctx context.Context, user *model.User, server *htt
 			{Name: "get_repo", Function: func(state *lua.State) int {
 				repo := lua.CheckString(l, 1)
 				reqURL, err := url.JoinPath("/repositories", repo)
+				if err != nil {
+					check(l, err)
+				}
 				req, err := newLakeFSJSONRequest(ctx, user, http.MethodGet, reqURL, nil)
 				if err != nil {
 					check(l, err)

From e5ca6b86baa2c38d2c3ddaaca0db3b014d15906f Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 14:58:19 +0200
Subject: [PATCH 23/35] pass a delta client to the export_delta function
 lua-require packages

---
 examples/hooks/delta_lake_S3_export.lua       | 14 ++++-------
 .../lakefs/catalogexport/delta_exporter.lua   | 23 +++++++++----------
 .../lua/lakefs/catalogexport/internal.lua     | 10 ++++++++
 3 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/examples/hooks/delta_lake_S3_export.lua b/examples/hooks/delta_lake_S3_export.lua
index e9e414f7508..d60d01e01fb 100644
--- a/examples/hooks/delta_lake_S3_export.lua
+++ b/examples/hooks/delta_lake_S3_export.lua
@@ -12,19 +12,13 @@
     storage_client:
         - put_object: function(bucket, key, data)
 ]]
-
-local export_delta_args = {
-    table_paths = args.table_paths,
-    lakefs_key = args.lakefs.access_key_id,
-    lakefs_secret = args.lakefs.secret_access_key,
-    region = args.aws.aws_region
-}
-
 local aws = require("aws")
-local delta = require("lakefs/catalogexport/delta_exporter")
+local delta_export = require("lakefs/catalogexport/delta_exporter")
+
 local sc = aws.s3_client(args.aws.aws_access_key_id, args.aws.aws_secret_access_key, args.aws.aws_region)
 
-local delta_table_locations = delta.export_delta_log(action, export_delta_args, sc)
+local delta_client = delta_export.get_delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.aws_region)
+local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc, delta_client)
 for t, loc in pairs(delta_table_locations) do
     print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n")
 end
diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index d144ff60507..36a6a594750 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -20,34 +20,32 @@ local function delta_log_entry_key_generator()
     end
 end
 
+local function get_delta_client(key, secret, region)
+    return formats.delta_client("s3", key, secret, region)
+end
+
 --[[
     action:
         - repository_id
         - commit_id
 
-    hook_args:
-        - table_paths: ["path/to/table1", "path/to/table2", ...]
-        - lakefs_key
-        - lakefs_secret
-        - region
+   table_paths: ["path/to/table1", "path/to/table2", ...]
 
     storage_client:
         - put_object: function(bucket, key, data)
 
+    delta_client:
+        - get_table: function(repo, ref, prefix)
+
 ]]
-local function export_delta_log(action, hook_args, storage_client)
+local function export_delta_log(action, table_paths, storage_client, delta)
     local repo = action.repository_id
     local commit_id = action.commit_id
-    local table_paths = hook_args.table_paths
-    local lakefs_key = hook_args.lakefs_key
-    local lakefs_secret = hook_args.lakefs_secret
-    local region = hook_args.region
 
     local ns = utils.get_storage_namespace(repo)
     if ns == nil then
         error("failed getting storage namespace for repo " .. repo)
     end
-    local delta = formats.delta_client("s3", lakefs_key, lakefs_secret, region)
     local response = {}
     for _, path in ipairs(table_paths) do
         local t = delta.get_table(repo, commit_id, path)
@@ -131,5 +129,6 @@ local function export_delta_log(action, hook_args, storage_client)
 end
 
 return {
-    export_delta_log = export_delta_log
+    export_delta_log = export_delta_log,
+    get_delta_client = get_delta_client
 }
diff --git a/pkg/actions/lua/lakefs/catalogexport/internal.lua b/pkg/actions/lua/lakefs/catalogexport/internal.lua
index d9bb4bf607d..231a47cbe2e 100644
--- a/pkg/actions/lua/lakefs/catalogexport/internal.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/internal.lua
@@ -1,4 +1,7 @@
 local url = require("net/url")
+local pathlib = require("path")
+local lakefs = require("lakefs")
+local json = require("encoding/json")
 local DEFAULT_SHORT_DIGEST_LEN=6
 
 local function deepcopy(orig)
@@ -66,6 +69,12 @@ local function parse_storage_uri(uri)
     }
 end
 
+local function get_storage_uri_prefix(storage_ns, commit_id, action_info)
+    local branch_or_tag = ref_from_branch_or_tag(action_info)
+    local sha = short_digest(commit_id)
+    return pathlib.join("/", storage_ns, string.format("_lakefs/exported/%s/%s/", branch_or_tag, sha))
+end
+
 local function sortedKeys(query, sortFunction)
     local keys, len = {}, 0
     for k,_ in pairs(query) do
@@ -95,4 +104,5 @@ return {
     lakefs_paginiated_api=lakefs_paginiated_api,
     sortedKeys = sortedKeys,
     get_storage_namespace = get_storage_namespace,
+    get_storage_uri_prefix = get_storage_uri_prefix,
 }

From 0b6853df1cec579b5d50cef855cc182be8acf4cb Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Thu, 30 Nov 2023 16:38:21 +0200
Subject: [PATCH 24/35] upgrade aws-sdk-v2

---
 go.mod | 4 ++--
 go.sum | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index cb4ac46c613..0575d668133 100644
--- a/go.mod
+++ b/go.mod
@@ -70,7 +70,7 @@ require (
 	github.com/alitto/pond v1.8.3
 	github.com/antonmedv/expr v1.15.3
 	github.com/aws/aws-sdk-go v1.45.25
-	github.com/aws/aws-sdk-go-v2 v1.21.2
+	github.com/aws/aws-sdk-go-v2 v1.23.4
 	github.com/aws/aws-sdk-go-v2/config v1.18.45
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.43
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.42
@@ -80,7 +80,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/glue v1.63.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2
 	github.com/aws/aws-sdk-go-v2/service/sts v1.23.2
-	github.com/aws/smithy-go v1.15.0
+	github.com/aws/smithy-go v1.18.1
 	github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc
 	github.com/csimplestring/delta-go v0.0.0-20231105162402-9b93ca02cedf
 	github.com/dgraph-io/badger/v4 v4.2.0
diff --git a/go.sum b/go.sum
index 78dfdb0dd36..f12eb3fa307 100644
--- a/go.sum
+++ b/go.sum
@@ -191,8 +191,9 @@ github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4
 github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4=
 github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
-github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA=
 github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
+github.com/aws/aws-sdk-go-v2 v1.23.4 h1:2P20ZjH0ouSAu/6yZep8oCmTReathLuEu6dwoqEgjts=
+github.com/aws/aws-sdk-go-v2 v1.23.4/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY=
@@ -260,8 +261,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4Nx
 github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU=
 github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ=
 github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
-github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
 github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c=
+github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/barweiss/go-tuple v1.1.1 h1:0dLwRqJHHpU9c5iJLF9Xjl5yEPDXwh5FhigydBoM5MM=
 github.com/barweiss/go-tuple v1.1.1/go.mod h1:oX3AtZyuGnCuK3B4PlUIfR9spxyJ42bgTSqCI9GkPB0=

From e32dc16c0a5aab1f539a5107c180603975938c7c Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 14:35:43 +0200
Subject: [PATCH 25/35] go mod tidy

---
 go.mod | 2 ++
 go.sum | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/go.mod b/go.mod
index a344d136795..d80dd8b34fd 100644
--- a/go.mod
+++ b/go.mod
@@ -107,6 +107,7 @@ require (
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
 	github.com/ahmetb/go-linq/v3 v3.2.0 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
+	github.com/aws/aws-sdk-go v1.44.314 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 // indirect
@@ -121,6 +122,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 // indirect
+	github.com/barweiss/go-tuple v1.1.1 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/deckarep/golang-set/v2 v2.3.1 // indirect
 	github.com/fraugster/parquet-go v0.12.0 // indirect
diff --git a/go.sum b/go.sum
index 7274a3560f5..003079a7e1e 100644
--- a/go.sum
+++ b/go.sum
@@ -188,6 +188,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.314 h1:d/5Jyk/Fb+PBd/4nzQg0JuC2W4A0knrDIzBgK/ggAow=
+github.com/aws/aws-sdk-go v1.44.314/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
 github.com/aws/aws-sdk-go-v2 v1.23.4 h1:2P20ZjH0ouSAu/6yZep8oCmTReathLuEu6dwoqEgjts=
 github.com/aws/aws-sdk-go-v2 v1.23.4/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
@@ -1080,6 +1082,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -1194,6 +1197,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1250,6 +1254,7 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1285,6 +1290,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
 golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1389,6 +1395,7 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
@@ -1401,6 +1408,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1477,6 +1485,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From aa476a8c8d12583648eb995073a9a73e426a579b Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 14:37:13 +0200
Subject: [PATCH 26/35] use get_storage_prefix from the internal package.
 Remove it from docs

---
 docs/howto/hooks/lua.md                       |  7 ----
 .../lakefs/catalogexport/symlink_exporter.lua | 39 +++++--------------
 2 files changed, 9 insertions(+), 37 deletions(-)

diff --git a/docs/howto/hooks/lua.md b/docs/howto/hooks/lua.md
index 6f542b64df8..e1f7428a0c4 100644
--- a/docs/howto/hooks/lua.md
+++ b/docs/howto/hooks/lua.md
@@ -403,13 +403,6 @@ local s3 = aws.s3_client(args.aws.aws_access_key_id, args.aws.aws_secret_access_
 exporter.export_s3(s3, args.table_descriptor_path, action, {debug=true})
 ```
 
-### `lakefs/catalogexport/symlink_exporter.get_storage_uri_prefix(storage_ns, commit_id, action_info)`
-
-Generate prefix for Symlink file(s) structure that represents a `ref` and a `commit` in lakeFS.
-The output pattern `${storage_ns}_lakefs/exported/${ref}/${commit_id}/`.
-The `ref` is deduced from the action event in `action_info` (i.e branch name).
-
-
 ### `lakefs/catalogexport/glue_exporter`
 
 A Package for automating the export process from lakeFS stored tables into Glue catalog.
diff --git a/pkg/actions/lua/lakefs/catalogexport/symlink_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/symlink_exporter.lua
index 457bb873ef6..202dee91143 100644
--- a/pkg/actions/lua/lakefs/catalogexport/symlink_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/symlink_exporter.lua
@@ -5,32 +5,12 @@ local pathlib = require("path")
 local strings = require("strings")
 local lakefs = require("lakefs")
 
---[[
-    ### Default Symlink File(s) structure:
-    
-    ${storageNamespace}
-    _lakefs/
-        exported/
-            ${ref}/
-                ${commitId}/
-                    ${tableName}/
-                        p1=v1/symlink.txt
-                        p1=v2/symlink.txt
-                        p1=v3/symlink.txt
-                        ...
-]]
-local function get_storage_uri_prefix(storage_ns, commit_id, action_info)
-    local branch_or_tag = utils.ref_from_branch_or_tag(action_info)
-    local sha = utils.short_digest(commit_id)
-    return pathlib.join("/", storage_ns, string.format("_lakefs/exported/%s/%s/", branch_or_tag, sha))
-end
-
 --[[
     @repo_id: repository id
     @commit_id: commit id of the current table
     @table_src_path: path to table spec (i.e _lakefs_tables/my_table.yaml)
     @options:
-    - skip_trim_obj_base_path(boolean) if true will skip removing the prefix path before the partition path. 
+    - skip_trim_obj_base_path(boolean) if true will skip removing the prefix path before the partition path.
 ]]
 local function export_it(repo_id, commit_id, table_src_path, options)
     local opts = options or {}
@@ -54,7 +34,7 @@ local function export_it(repo_id, commit_id, table_src_path, options)
         for _, entry in ipairs(entries) do
             symlink_data = symlink_data .. entry.physical_address .. "\n"
         end
-        -- create key suffix for symlink file 
+        -- create key suffix for symlink file
         local storage_key_suffix = part_key
         if #descriptor.partition_columns == 0 then
             storage_key_suffix = descriptor.name .. "/" .. "symlink.txt"
@@ -62,7 +42,7 @@ local function export_it(repo_id, commit_id, table_src_path, options)
             if not opts.skip_trim_obj_base_path then
                 storage_key_suffix = strings.replace(part_key, base_path .. "/", "", 1) -- remove base_path prefix from partition path
             end
-            -- append to partition path to suffix 
+            -- append to partition path to suffix
             storage_key_suffix = pathlib.join("/", descriptor.name, storage_key_suffix, "symlink.txt")
         end
         return {
@@ -73,13 +53,13 @@ local function export_it(repo_id, commit_id, table_src_path, options)
 end
 
 --[[
-    export a Symlinks that represent a table to S3 
-    @s3_client: configured client 
+    export a Symlinks that represent a table to S3
+    @s3_client: configured client
     @table_src_path: object path to the table spec (_lakefs_tables/my_table.yaml)
-    @action_info: the global action object 
-    @options: 
+    @action_info: the global action object
+    @options:
     - debug(boolean)
-    - export_base_uri(string): override the prefix in S3 i.e s3://other-bucket/path/ 
+    - export_base_uri(string): override the prefix in S3 i.e s3://other-bucket/path/
     - writer(function(bucket, key, data)): if passed then will not use s3 client, helpful for debug
 ]]
 local function export_s3(s3_client, table_src_path, action_info, options)
@@ -87,7 +67,7 @@ local function export_s3(s3_client, table_src_path, action_info, options)
     local repo_id = action_info.repository_id
     local commit_id = action_info.commit_id
     local base_prefix = opts.export_base_uri or action_info.storage_namespace
-    local export_base_uri = get_storage_uri_prefix(base_prefix, commit_id, action_info)
+    local export_base_uri = utils.get_storage_uri_prefix(base_prefix, commit_id, action_info)
     local location = utils.parse_storage_uri(export_base_uri)
     local put_object = opts.writer or s3_client.put_object
     local it = export_it(repo_id, commit_id, table_src_path, opts)
@@ -105,5 +85,4 @@ end
 
 return {
     export_s3 = export_s3,
-    get_storage_uri_prefix=get_storage_uri_prefix,
 }

From 0bae4ba6d10389a9a63fd127bc75f9568b546d5d Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 14:39:17 +0200
Subject: [PATCH 27/35] remove get_storage_namespace and the `get_repo` method
 from lakeFS client since the storage namespace can be fetched from the global
 action object

---
 .../lakefs/catalogexport/delta_exporter.lua   |  2 +-
 .../lua/lakefs/catalogexport/internal.lua     | 10 ----------
 pkg/actions/lua/lakefs/client.go              | 19 -------------------
 3 files changed, 1 insertion(+), 30 deletions(-)

diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index 36a6a594750..aa378c6ba96 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -42,7 +42,7 @@ local function export_delta_log(action, table_paths, storage_client, delta)
     local repo = action.repository_id
     local commit_id = action.commit_id
 
-    local ns = utils.get_storage_namespace(repo)
+    local ns = action.storage_namespace
     if ns == nil then
         error("failed getting storage namespace for repo " .. repo)
     end
diff --git a/pkg/actions/lua/lakefs/catalogexport/internal.lua b/pkg/actions/lua/lakefs/catalogexport/internal.lua
index 231a47cbe2e..7125cab9b56 100644
--- a/pkg/actions/lua/lakefs/catalogexport/internal.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/internal.lua
@@ -86,15 +86,6 @@ local function sortedKeys(query, sortFunction)
     return keys
 end
 
-local function get_storage_namespace(repo)
-    local code, obj = lakefs.get_repo(repo)
-    if code == 200 then
-        local obj_repo = json.unmarshal(obj)
-        return obj_repo["storage_namespace"]
-    end
-    return nil
-end
-
 return {
     deepcopy=deepcopy,
     parse_storage_uri=parse_storage_uri,
@@ -103,6 +94,5 @@ return {
     lakefs_object_pager=lakefs_object_pager,
     lakefs_paginiated_api=lakefs_paginiated_api,
     sortedKeys = sortedKeys,
-    get_storage_namespace = get_storage_namespace,
     get_storage_uri_prefix = get_storage_uri_prefix,
 }
diff --git a/pkg/actions/lua/lakefs/client.go b/pkg/actions/lua/lakefs/client.go
index cae9d3e5186..ea432d47f1b 100644
--- a/pkg/actions/lua/lakefs/client.go
+++ b/pkg/actions/lua/lakefs/client.go
@@ -204,25 +204,6 @@ func OpenClient(l *lua.State, ctx context.Context, user *model.User, server *htt
 				l.PushString(rr.Body.String())
 				return 2
 			}},
-			{Name: "get_repo", Function: func(state *lua.State) int {
-				repo := lua.CheckString(l, 1)
-				reqURL, err := url.JoinPath("/repositories", repo)
-				if err != nil {
-					check(l, err)
-				}
-				req, err := newLakeFSJSONRequest(ctx, user, http.MethodGet, reqURL, nil)
-				if err != nil {
-					check(l, err)
-				}
-				// query params
-				q := req.URL.Query()
-				req.URL.RawQuery = q.Encode()
-				rr := httptest.NewRecorder()
-				server.Handler.ServeHTTP(rr, req)
-				l.PushInteger(rr.Code)
-				l.PushString(rr.Body.String())
-				return 2
-			}},
 			{Name: "diff_branch", Function: func(state *lua.State) int {
 				repo := lua.CheckString(l, 1)
 				branch := lua.CheckString(l, 2)

From a603196733871d59c401e4adfd9338fd86771023 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 14:39:36 +0200
Subject: [PATCH 28/35] use get_storage_prefix from the internal package

---
 .../lua/lakefs/catalogexport/glue_exporter.lua   | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/pkg/actions/lua/lakefs/catalogexport/glue_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/glue_exporter.lua
index 056a06a9122..786eef88f02 100644
--- a/pkg/actions/lua/lakefs/catalogexport/glue_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/glue_exporter.lua
@@ -8,7 +8,7 @@ local sym_exporter = require("lakefs/catalogexport/symlink_exporter")
 --[[
     Generate glue table name
     @descriptor(Table): object from (i.e _lakefs_tables/my_table.yaml)
-    @action_info(Table): the global action object 
+    @action_info(Table): the global action object
 ]]
 local function get_full_table_name(descriptor, action_info)
     local commit_id = action_info.commit_id
@@ -33,7 +33,7 @@ local function hive_col_to_glue(col)
     }
 end
 
--- Create list of partitions for Glue input from a Hive descriptor 
+-- Create list of partitions for Glue input from a Hive descriptor
 local function hive_partitions_to_glue_input(descriptor)
     local partitions = {}
     local cols = descriptor.schema.fields or {}
@@ -54,16 +54,16 @@ end
 
 -- Create list of columns for Glue excluding partitions
 local function hive_columns_to_glue_input(descriptor)
-    -- create set of partition names since they must not appear in the columns input in glue 
+    -- create set of partition names since they must not appear in the columns input in glue
     local partition_names = {}
     for _, p in ipairs(descriptor.partition_columns) do
         partition_names[p] = true
     end
-    -- create columns as inputs for glue 
+    -- create columns as inputs for glue
     local columns = {}
     local cols = descriptor.schema.fields or {}
     for _, col in ipairs(cols) do
-        if not partition_names[col.name] then -- not a partition 
+        if not partition_names[col.name] then -- not a partition
             table.insert(columns, hive_col_to_glue(col))
         end
     end
@@ -73,7 +73,7 @@ end
 -- default location value (e.g root location of either partitions or flat symlink.txt file)
 local function get_table_location(storage_base_prefix, descriptor, action_info)
     local commit_id = action_info.commit_id
-    local export_base_uri = sym_exporter.get_storage_uri_prefix(storage_base_prefix, commit_id, action_info)
+    local export_base_uri = utils.get_storage_uri_prefix(storage_base_prefix, commit_id, action_info)
     return pathlib.join("/", export_base_uri, descriptor.name)
 end
 
@@ -96,7 +96,7 @@ end
     @db(string): glue database name
     @table_src_path(string): path to table spec (i.e _lakefs_tables/my_table.yaml)
     @create_table_input(Table): struct mapping to table_input in AWS https://docs.aws.amazon.com/glue/latest/webapi/API_CreateTable.html#API_CreateTable_RequestSyntax
-    should contain inputs describing the data format (i.e InputFormat, OutputFormat, SerdeInfo) since the exporter is agnostic to this. 
+    should contain inputs describing the data format (i.e InputFormat, OutputFormat, SerdeInfo) since the exporter is agnostic to this.
     by default this function will configure table location and schema.
     @action_info(Table): the global action object
     @options:
@@ -112,7 +112,7 @@ local function export_glue(glue, db, table_src_path, create_table_input, action_
     -- get table desctiptor from _lakefs_tables/
     local descriptor = extractor.get_table_descriptor(lakefs, repo_id, commit_id, table_src_path)
 
-    -- get table symlink location uri 
+    -- get table symlink location uri
     local base_prefix = opts.export_base_uri or action_info.storage_namespace
     local symlink_location = get_table_location(base_prefix, descriptor, action_info)
 

From 9e522f552b99ae738e8002a1090652d14f71c1b7 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 15:04:08 +0200
Subject: [PATCH 29/35] use correct error when returning

---
 pkg/actions/lua/formats/delta.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index b82b4f171a7..3a72719fdb8 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -69,7 +69,7 @@ func (dc *DeltaClient) buildLog(table delta.Log) (map[int64][]string, error) {
 		entryVersion := entry.Version()
 		actions, aErr := entry.Actions()
 		if aErr != nil {
-			return nil, err
+			return nil, aErr
 		}
 		for _, a := range actions {
 			aj, _ := a.Json()

From 178107dc7be9700dc017ffd85e5dc4406031efef Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 15:05:03 +0200
Subject: [PATCH 30/35] comment the delta_log_entry_key_generator function and
 return a complete key name (including ".json" suffix)

---
 .../lua/lakefs/catalogexport/delta_exporter.lua      | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index aa378c6ba96..4c9fadd1e72 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -4,6 +4,14 @@ local pathlib = require("path")
 local json = require("encoding/json")
 local utils = require("lakefs/catalogexport/internal")
 
+--[[
+    delta_log_entry_key_generator returns a closure that returns a Delta Lake version key according to the Delta Lake
+    protocol: https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries
+    Example:
+        local gen = delta_log_entry_key_generator()
+        gen() -- 000000000000000001.json
+        gen() -- 000000000000000002.json
+]]
 local function delta_log_entry_key_generator()
     local current = 0
     return function()
@@ -14,7 +22,7 @@ local function delta_log_entry_key_generator()
         for _ = 1, padding_length do
             padded_key = padded_key .. "0"
         end
-        padded_key = padded_key .. key
+        padded_key = padded_key .. key .. ".json"
         current = current + 1
         return padded_key
     end
@@ -93,7 +101,7 @@ local function export_delta_log(action, table_paths, storage_client, delta)
                 local entry_m = json.marshal(entry)
                 table.insert(entry_log, entry_m)
             end
-            table_log[string.format("%s.json", keyGenerator())] = entry_log
+            table_log[keyGenerator()] = entry_log
         end
 
         -- Get the table delta log physical location

From a687c9132f29cf142b96d18ec921e906ab070a78 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 15:16:53 +0200
Subject: [PATCH 31/35] use a writer_client function instead of a storage
 client when passed to the export_delta_log function

---
 examples/hooks/delta_lake_S3_export.lua                 | 2 +-
 pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/examples/hooks/delta_lake_S3_export.lua b/examples/hooks/delta_lake_S3_export.lua
index d60d01e01fb..6899be15795 100644
--- a/examples/hooks/delta_lake_S3_export.lua
+++ b/examples/hooks/delta_lake_S3_export.lua
@@ -18,7 +18,7 @@ local delta_export = require("lakefs/catalogexport/delta_exporter")
 local sc = aws.s3_client(args.aws.aws_access_key_id, args.aws.aws_secret_access_key, args.aws.aws_region)
 
 local delta_client = delta_export.get_delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.aws_region)
-local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc, delta_client)
+local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc.put_object, delta_client)
 for t, loc in pairs(delta_table_locations) do
     print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n")
 end
diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index 4c9fadd1e72..c37b6139f84 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -39,14 +39,13 @@ end
 
    table_paths: ["path/to/table1", "path/to/table2", ...]
 
-    storage_client:
-        - put_object: function(bucket, key, data)
+    write_object: function(bucket, key, data)
 
     delta_client:
         - get_table: function(repo, ref, prefix)
 
 ]]
-local function export_delta_log(action, table_paths, storage_client, delta)
+local function export_delta_log(action, table_paths, write_object, delta)
     local repo = action.repository_id
     local commit_id = action.commit_id
 
@@ -129,7 +128,7 @@ local function export_delta_log(action, table_paths, storage_client, delta)
                 table_entry_string = table_entry_string .. content_entry
             end
             local version_key = storage_props.key .. "/" .. entry_version
-            storage_client.put_object(storage_props.bucket, version_key, table_entry_string)
+            write_object(storage_props.bucket, version_key, table_entry_string)
         end
         response[t_name] = table_physical_path
     end

From 98c3ac2d68d7c31105d4b39f585c65ad595a311b Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 15:44:23 +0200
Subject: [PATCH 32/35] updated version of delta go

---
 go.mod | 107 ++++++++++++++-------------
 go.sum | 222 ++++++++++++++++++++++++++++++---------------------------
 2 files changed, 174 insertions(+), 155 deletions(-)

diff --git a/go.mod b/go.mod
index d80dd8b34fd..43903a2ffa6 100644
--- a/go.mod
+++ b/go.mod
@@ -3,15 +3,15 @@ module github.com/treeverse/lakefs
 go 1.21
 
 require (
-	cloud.google.com/go v0.110.8 // indirect
-	cloud.google.com/go/storage v1.33.0
+	cloud.google.com/go v0.111.0 // indirect
+	cloud.google.com/go/storage v1.35.1
 	github.com/apache/thrift v0.19.0
 	github.com/cockroachdb/pebble v0.0.0-20230106151110-65ff304d3d7a
 	github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/deepmap/oapi-codegen v1.5.6
 	github.com/dgraph-io/ristretto v0.1.1
-	github.com/fsnotify/fsnotify v1.6.0
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/getkin/kin-openapi v0.53.0
 	github.com/go-chi/chi/v5 v5.0.10
 	github.com/go-openapi/swag v0.19.14
@@ -19,7 +19,7 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.4.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hnlq715/golang-lru v0.3.0
 	github.com/jamiealquiza/tachymeter v2.0.0+incompatible
@@ -45,23 +45,23 @@ require (
 	github.com/vbauerster/mpb/v5 v5.4.0
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20230607234618-40034c8066df
-	golang.org/x/crypto v0.14.0
-	golang.org/x/oauth2 v0.13.0
-	golang.org/x/term v0.13.0
-	google.golang.org/api v0.147.0
+	golang.org/x/crypto v0.16.0
+	golang.org/x/oauth2 v0.15.0
+	golang.org/x/term v0.15.0
+	google.golang.org/api v0.152.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
-	golang.org/x/sync v0.4.0
+	cloud.google.com/go/compute v1.23.3 // indirect
+	golang.org/x/sync v0.5.0
 )
 
 require (
 	cloud.google.com/go/compute/metadata v0.2.3
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
 	github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
@@ -69,16 +69,16 @@ require (
 	github.com/Shopify/go-lua v0.0.0-20221004153744-91867de107cf
 	github.com/alitto/pond v1.8.3
 	github.com/antonmedv/expr v1.15.3
-	github.com/aws/aws-sdk-go-v2 v1.23.4
-	github.com/aws/aws-sdk-go-v2/config v1.25.10
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.8
+	github.com/aws/aws-sdk-go-v2 v1.23.5
+	github.com/aws/aws-sdk-go-v2/config v1.25.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.9
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.7
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.7
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.3
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.26.1
 	github.com/aws/aws-sdk-go-v2/service/glue v1.71.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.2
 	github.com/aws/smithy-go v1.18.1
 	github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc
 	github.com/csimplestring/delta-go v0.0.0-20231105162402-9b93ca02cedf
@@ -99,40 +99,42 @@ require (
 )
 
 require (
-	cloud.google.com/go/iam v1.1.2 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/ahmetb/go-linq/v3 v3.2.0 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
-	github.com/aws/aws-sdk-go v1.44.314 // indirect
+	github.com/aws/aws-sdk-go v1.48.11 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 // indirect
-	github.com/barweiss/go-tuple v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 // indirect
+	github.com/barweiss/go-tuple v1.1.2 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
-	github.com/deckarep/golang-set/v2 v2.3.1 // indirect
+	github.com/deckarep/golang-set/v2 v2.5.0 // indirect
 	github.com/fraugster/parquet-go v0.12.0 // indirect
 	github.com/getsentry/sentry-go v0.16.0 // indirect
-	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
-	github.com/golang/glog v1.1.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/golang/glog v1.1.2 // indirect
 	github.com/google/flatbuffers v2.0.0+incompatible // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
@@ -151,16 +153,20 @@ require (
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/samber/mo v1.10.0 // indirect
+	github.com/samber/mo v1.11.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	gocloud.dev v0.34.0 // indirect
+	gocloud.dev v0.34.1-0.20231122211418-53ccd8db26a1 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	gonum.org/v1/gonum v0.9.3 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
 )
 
 require (
@@ -190,7 +196,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -233,18 +238,18 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.11.0
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.17.0
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/grpc v1.58.3
+	golang.org/x/exp v0.0.0-20231127185646-65229373498e
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 // indirect
+	google.golang.org/grpc v1.59.0
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
-replace github.com/csimplestring/delta-go => github.com/treeverse/delta-go v0.0.0-20231128165607-87acc8332e05
+replace github.com/csimplestring/delta-go => github.com/treeverse/delta-go v0.0.0-20231203131847-a5acc36c8ba5
diff --git a/go.sum b/go.sum
index 003079a7e1e..95943d7a163 100644
--- a/go.sum
+++ b/go.sum
@@ -31,8 +31,8 @@ cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Ud
 cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
 cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U=
 cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
-cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME=
-cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk=
+cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM=
+cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -43,8 +43,8 @@ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTB
 cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw=
 cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
 cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
-cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
-cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
+cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
@@ -53,8 +53,8 @@ cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx
 cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c=
 cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw=
 cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
-cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4=
-cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
+cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
+cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
 cloud.google.com/go/kms v1.1.0/go.mod h1:WdbppnCDMDpOvoYBMn1+gNmOeEoZYqAv+HeuKARGCXI=
 cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA=
 cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4=
@@ -72,8 +72,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA=
-cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
-cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
+cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w=
+cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
 cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A=
 cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM=
 contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
@@ -92,8 +92,8 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
@@ -103,8 +103,8 @@ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6 h1:oBqQLSI1pZwGOdXJAo
 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6/go.mod h1:Beh5cHIXJ0oWEDWk9lNFtuklCojLLQ5hl+LqSNTTs0I=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA=
@@ -136,8 +136,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -188,41 +188,41 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.44.314 h1:d/5Jyk/Fb+PBd/4nzQg0JuC2W4A0knrDIzBgK/ggAow=
-github.com/aws/aws-sdk-go v1.44.314/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.48.11 h1:9YbiSbaF/jWi+qLRl+J5dEhr2mcbDYHmKg2V7RBcD5M=
+github.com/aws/aws-sdk-go v1.48.11/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
-github.com/aws/aws-sdk-go-v2 v1.23.4 h1:2P20ZjH0ouSAu/6yZep8oCmTReathLuEu6dwoqEgjts=
-github.com/aws/aws-sdk-go-v2 v1.23.4/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
+github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg=
+github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA=
 github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg=
-github.com/aws/aws-sdk-go-v2/config v1.25.10 h1:qw/e8emDtNufTkrAU86DlQ18DruMyyM7ttW6Lgwp4v0=
-github.com/aws/aws-sdk-go-v2/config v1.25.10/go.mod h1:203YiAtb6XyoGxXMPsUVwEcuxCiTQY/r8P27IDjfvMc=
+github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno=
+github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s=
 github.com/aws/aws-sdk-go-v2/credentials v1.11.2/go.mod h1:j8YsY9TXTm31k4eFhspiQicfXPLZ0gYXA50i4gxPE8g=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.8 h1:phw9nRLy/77bPk6Mfu2SHCOnHwfVB7WWrOa5rZIY2Fc=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.8/go.mod h1:MrS4SOin6adbO6wgWhdifyPiq+TX7fPPwyA/ZLC1F5M=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.9/go.mod h1:R7mDuIJoCjH6TxGUc/cylE7Lp/o0bhKVoxdBThsjqCM=
 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.7 h1:UPEINjKdlj8AqwBjtW6x6b074Bfi+mYUy6U+fiUBdb0=
 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.7/go.mod h1:Q+NAskmU4CCnGQxJOiXRU4ww71B8syJxPQ5QDWMbxX0=
 github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.7 h1:bW5XJXfakohTCCpxB/udKiUQKPthoUPJjmqXETjwdoM=
 github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.7/go.mod h1:0coJJPClBzU6XXpywvxpVl4y8ELMIveAAgR0CV2VhBE=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3/go.mod h1:uk1vhHHERfSVCUnqSqz8O48LBYDSC+k6brng09jcMOk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 h1:tQZLSPC2Zj2CqZHonLmWEvCsbpMX5tQvaYJWHadcPek=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8/go.mod h1:5+YpvTHDFffykWr5qAGjqwoh8oVYZOddL3sSrEN7lws=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.3/go.mod h1:0dHuD2HZZSiwfJSy1FO5bX1hQ1TxVV1QXXjpn3XUE44=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.3 h1:0Pw2ku539I0EugduMpJ+579WRc+38nv8rZhThWjsuYQ=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.3/go.mod h1:vQtGu6huTQkoEhNgkDeijtYm9Y8HgpQqvGeKUPoEunY=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4 h1:TUCNKBd4/JEefsZDxo5deRmrRRPZHqGyBYiUAeBKOWU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4/go.mod h1:egDkcl+zsgFqS6VO142bKboip5Pe1sNMwN55Xy38QsM=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 h1:eMqD7ku6WGdmcWWXPYun9m6yk6feSULLhJlAtN6rYG4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7/go.mod h1:0oBIfcDV6LScxEW0VgOqxT3e4aqKRp+SYhB9wAd5E3Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7 h1:+XYhWhgWs5F3Zx8oa49CXzNvfXrItaDjZB/M172fcHQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7/go.mod h1:L6tcSRyCGxcKfDWUrmv2jv8G1cLDU7d0FUpEFpG9bVE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10/go.mod h1:8DcYQcz0+ZJaSxANlHIsbbi6S+zMwjwdDqwW3r9AzaE=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7 h1:3VaUNB1LclLomv82VnP5QnxAfowG+Ro4m82+af9wjZ4=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7/go.mod h1:D5i0c+qvEY0LV5F4elFZd+mYnvHQbufCLHNHoBfQR2g=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4=
 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.26.1 h1:F+gzyWzrS53QtGXxX/d+kerN1DJMeFEPsyZW4E33hUA=
 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.26.1/go.mod h1:QQQ9e20uyRCDABPb6Cw729u1XIbH3LBIc9QsCCcDOBo=
 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.1 h1:vitPmwVNVZm/3aWsOQJj+/9ZuHiNEz/kyEpFPJ3Wlqo=
@@ -233,38 +233,38 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:G
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.3/go.mod h1:Seb8KNmD6kVTjwRjVEgOT5hPin6sq+v4C2ycJQDwuH8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7 h1:Mft1tmIK1fkFS9l9sYVYiN+OdgXeOcQ9ZS3SxKOh3A4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7/go.mod h1:QWI83fhocxDaN3b74N8rrvET60CBaike5lQ+5sm3OcE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI=
 github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.7 h1:vxWJnVqDMv5F6rHHAJ+XsZQDCa8KmUlr3xSOPN3uxUc=
 github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.7/go.mod h1:s1pcqNgty0l9w56NUljd4HDTFRlzp2MsiyrrDjucE0I=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3/go.mod h1:wlY6SVjuwvh3TVRpTqdy4I1JpBFLX4UGeKZdWntaocw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7 h1:dU+ZyhvqMB/T/TxjGagHMCdyUiqaThRIaMu3YvKiSQI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7/go.mod h1:SGORuNqoXyWfTvTp/gBGJfv8jRvW/+nha0XhnIXVI+o=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.3/go.mod h1:Bm/v2IaN6rZ+Op7zX+bOUMdL4fsrYZiD0dsjLhNKwZc=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 h1:ybtGXm0qFVFi0hFUF7eFAVnL3ntl9MO7lrxhhGP7KYU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7/go.mod h1:BUyWJUKAnNqoEq1LfyQxy+Eh4U8Y3c5w2C6m21f3yvI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c=
 github.com/aws/aws-sdk-go-v2/service/kms v1.16.3/go.mod h1:QuiHPBqlOFCi4LqdSskYYAWpQlx3PKmohy+rE2F+o5g=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3/go.mod h1:g1qvDuRsJY+XghsV6zg00Z4KJ7DtFFCx8fJD2a491Ak=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1 h1:0/W5F+LlXzKZ7KTsRcD8pugasVnsrjUWmhOsN/LdSFY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1/go.mod h1:TqThLn4bRCn/UYf960hNZgPPjmxc17fQcwmjfuG6D5k=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2 h1:DLSAG8zpJV2pYsU+UPkj1IEZghyBnnUsvIRs6UuXSDU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.4/go.mod h1:PJc8s+lxyU8rrre0/4a0pn2wgwiDvOEzoOjcJUBr67o=
 github.com/aws/aws-sdk-go-v2/service/sns v1.17.4/go.mod h1:kElt+uCcXxcqFyc+bQqZPFD9DME/eC6oHBXvFzQ9Bcw=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3/go.mod h1:skmQo0UPvsjsuYYSYMVmrPc1HWCbHUJyrCEp+ZaLzqM=
 github.com/aws/aws-sdk-go-v2/service/ssm v1.24.1/go.mod h1:NR/xoKjdbRJ+qx0pMR4mI+N/H1I1ynHwXnO6FowXJc0=
 github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoHYgRmgfGM4mgsLYtcObdveU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 h1:V40g2daNO3l1J94JYwqfkyvQMYXi5I25fs3fNQW8iDs=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.1/go.mod h1:0ZWQJP/mBOUxkCvZKybZNz1XmdUKSBxoF0dzgfxtvDs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 h1:uQrj7SpUNC3r55vc1CDh3qV9wJC66lz546xM9dhSo5s=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1/go.mod h1:oyaTk5xEAOuPXX1kCD7HmIeuLqdj3Bk5yGkqGXtGi14=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 h1:xJPydhNm0Hiqct5TVKEuHG7weC0+sOs4MUnd7A5n5F4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.2/go.mod h1:zxk6y1X2KXThESWMS5CrKRvISD8mbIMab6nZrCGxDG0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 h1:8dU9zqA77C5egbU6yd4hFLaiIdPv3rU+6cp7sz5FjCU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.1 h1:K33V7L0XDdb23FMOZySr8bon1jou5SHn1fiv7NJ1SUg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.1/go.mod h1:YtXUl/sfnS06VksYhr855hTQf2HphfT1Xv/EwuzbPjg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 h1:fFrLsy08wEbAisqW3KDl/cPHrF43GmV79zXB9EwJiZw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.2/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8=
 github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
 github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c=
 github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
-github.com/barweiss/go-tuple v1.1.1 h1:0dLwRqJHHpU9c5iJLF9Xjl5yEPDXwh5FhigydBoM5MM=
-github.com/barweiss/go-tuple v1.1.1/go.mod h1:oX3AtZyuGnCuK3B4PlUIfR9spxyJ42bgTSqCI9GkPB0=
+github.com/barweiss/go-tuple v1.1.2 h1:ul9tIW0LZ5w+Vk/Hi3X9z3JyqkD0yaVGZp+nNTLW2YE=
+github.com/barweiss/go-tuple v1.1.2/go.mod h1:SpoVilkI7ycNrIkQxcQfS1JG5A+R40sWwEUlPONlp3k=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -340,8 +340,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A=
-github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/deckarep/golang-set/v2 v2.5.0 h1:hn6cEZtQ0h3J8kFrHR/NrzyOoTnjgW1+FmNJzQ7y/sA=
+github.com/deckarep/golang-set/v2 v2.5.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/deepmap/oapi-codegen v1.5.6 h1:hlUtA13SL2HNoC/5vXDFdGm2AaN1j9/rUu7zedjBiqg=
 github.com/deepmap/oapi-codegen v1.5.6/go.mod h1:NoliSkZp7cRAkisw65+PUtvgy56f21QQesX1tVUzS8g=
 github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU=
@@ -400,8 +400,8 @@ github.com/fraugster/parquet-go v0.12.0/go.mod h1:dGzUxdNqXsAijatByVgbAWVPlFirnh
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
-github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
-github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU=
 github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38=
@@ -438,6 +438,11 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -480,14 +485,14 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
 github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
-github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
+github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
-github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
+github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
+github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -547,8 +552,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE=
 github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk=
@@ -590,12 +595,13 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
 github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
@@ -967,8 +973,8 @@ github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9c
 github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
-github.com/samber/mo v1.10.0 h1:dxwUGzd5i2FyPQJZzTIdexUuQC6WhXlq7zUJ9o7XCqI=
-github.com/samber/mo v1.10.0/go.mod h1:BfkrCPuYzVG3ZljnZB783WIJIGk1mcZr9c9CPf8tAxs=
+github.com/samber/mo v1.11.0 h1:ZOiSkrGGpNhVv/1dxP02risztdMTIwE8KSW9OG4k5bY=
+github.com/samber/mo v1.11.0/go.mod h1:BfkrCPuYzVG3ZljnZB783WIJIGk1mcZr9c9CPf8tAxs=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
@@ -1036,8 +1042,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/thanhpk/randstr v1.0.6 h1:psAOktJFD4vV9NEVb3qkhRSMvYh4ORRaj1+w/hn4B+o=
 github.com/thanhpk/randstr v1.0.6/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U=
-github.com/treeverse/delta-go v0.0.0-20231128165607-87acc8332e05 h1:2J9ZsK33cYZRANUjEeYYqPPlv3kA9ETTnbUyqCd/IYQ=
-github.com/treeverse/delta-go v0.0.0-20231128165607-87acc8332e05/go.mod h1:KUVLx8Ya8UkX9BZSx2MehNINLdFYyLuvsaNXWU1k208=
+github.com/treeverse/delta-go v0.0.0-20231203131847-a5acc36c8ba5 h1:ef28c0SQH9dac7JLyXX5eO66vIqEboJjLBPq+TtgMsA=
+github.com/treeverse/delta-go v0.0.0-20231203131847-a5acc36c8ba5/go.mod h1:E7uPCvF9rw8UQt6uDMN05snxpD45/I/UXAZxzVIYTgI=
 github.com/tsenart/vegeta/v12 v12.11.1 h1:Rbwe7Zxr7sJ+BDTReemeQalYPvKiSV+O7nwmUs20B3E=
 github.com/tsenart/vegeta/v12 v12.11.1/go.mod h1:swiFmrgpqj2llHURgHYFRFN0tfrIrlnspg01HjwOnSQ=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@@ -1094,6 +1100,14 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
+go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
+go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
+go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o=
+go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
+go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
+go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -1119,8 +1133,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg=
-gocloud.dev v0.34.0 h1:LzlQY+4l2cMtuNfwT2ht4+fiXwWf/NmPTnXUlLmGif4=
-gocloud.dev v0.34.0/go.mod h1:psKOachbnvY3DAOPbsFVmLIErwsbWPUG2H5i65D38vE=
+gocloud.dev v0.34.1-0.20231122211418-53ccd8db26a1 h1:ndqA6w+otk9a4nmdepcA9exfqXHgAw5S/55Gg1KwYv4=
+gocloud.dev v0.34.1-0.20231122211418-53ccd8db26a1/go.mod h1:wbyF+BhfdtLWyUtVEWRW13hFLb1vXnV2ovEhYGQe3ck=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -1146,8 +1160,8 @@ golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1162,8 +1176,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -1198,8 +1212,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1254,9 +1268,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1277,8 +1290,8 @@ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1291,8 +1304,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1385,20 +1398,18 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1408,9 +1419,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1418,6 +1429,8 @@ golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1486,16 +1499,16 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
+golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
@@ -1547,16 +1560,17 @@ google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7
 google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
 google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
 google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
-google.golang.org/api v0.147.0 h1:Can3FaQo9LlVqxJCodNmeZW/ib3/qKAY3rFeXiHo5gc=
-google.golang.org/api v0.147.0/go.mod h1:pQ/9j83DcmPd/5C9e2nFOdjjNkDZ1G+zkbK2uvdkJMs=
+google.golang.org/api v0.152.0 h1:t0r1vPnfMc260S2Ci+en7kfCZaLOPs5KI0sVV/6jZrY=
+google.golang.org/api v0.152.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1642,12 +1656,12 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2
 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
 google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
 google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=
-google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk=
-google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU=
-google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0=
+google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 h1:W12Pwm4urIbRdGhMEg2NM9O3TWKjNcxQhs46V0ypf/k=
+google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic=
+google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM=
 google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1677,8 +1691,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
-google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

From 590f2e305c393473ce5c9a5a729705d95ff8d179 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Sun, 3 Dec 2023 15:53:41 +0200
Subject: [PATCH 33/35] severAddress -> lakeFSAddr + comment

---
 pkg/actions/lua/formats/delta.go   | 15 ++++++++-------
 pkg/actions/lua/formats/formats.go |  4 ++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index 3a72719fdb8..f4fa98b6a96 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -123,18 +123,19 @@ func (awsI AWSInfo) GetAccessProperties() (interface{}, error) {
 }
 
 // newDelta is a factory function to create server/cloud specific Delta Lake client
-func newDelta(ctx context.Context, serverAddress string) lua.Function {
-	if strings.HasPrefix(serverAddress, ":") {
+// lakeFSAddr is the domain or "authority:port" of the running lakeFS server
+func newDelta(ctx context.Context, lakeFSAddr string) lua.Function {
+	if strings.HasPrefix(lakeFSAddr, ":") {
 		// workaround in case we listen on all interfaces without specifying ip
-		serverAddress = fmt.Sprintf("localhost%s", serverAddress)
+		lakeFSAddr = fmt.Sprintf("localhost%s", lakeFSAddr)
 	}
-	serverAddress = fmt.Sprintf("http://%s", serverAddress)
+	lakeFSAddr = fmt.Sprintf("http://%s", lakeFSAddr)
 	return func(l *lua.State) int {
 		var client *DeltaClient
 		st := lua.CheckString(l, 1)
 		switch storageType(st) {
 		case s3StorageType:
-			client = newS3DeltaClient(l, ctx, serverAddress)
+			client = newS3DeltaClient(l, ctx, lakeFSAddr)
 		default:
 			lua.Errorf(l, "unimplemented storage type")
 			panic("unimplemented storage type")
@@ -148,7 +149,7 @@ func newDelta(ctx context.Context, serverAddress string) lua.Function {
 	}
 }
 
-func newS3DeltaClient(l *lua.State, ctx context.Context, serverAddress string) *DeltaClient {
+func newS3DeltaClient(l *lua.State, ctx context.Context, lakeFSAddr string) *DeltaClient {
 	accessKeyID := lua.CheckString(l, 2)
 	secretAccessKey := lua.CheckString(l, 3)
 	r := lua.CheckString(l, 4)
@@ -161,7 +162,7 @@ func newS3DeltaClient(l *lua.State, ctx context.Context, serverAddress string) *
 				SecretAccessKey: secretAccessKey,
 			}, nil
 		}),
-		Endpoint: serverAddress,
+		Endpoint: lakeFSAddr,
 	}
 	storage.RegisterS3CompatBucketURLOpener("lakefs", &awsProps)
 
diff --git a/pkg/actions/lua/formats/formats.go b/pkg/actions/lua/formats/formats.go
index ab9bcaf9605..cd94fb647b4 100644
--- a/pkg/actions/lua/formats/formats.go
+++ b/pkg/actions/lua/formats/formats.go
@@ -6,10 +6,10 @@ import (
 	"github.com/Shopify/go-lua"
 )
 
-func Open(l *lua.State, ctx context.Context, serverAddress string) {
+func Open(l *lua.State, ctx context.Context, lakeFSAddr string) {
 	open := func(l *lua.State) int {
 		lua.NewLibrary(l, []lua.RegistryFunction{
-			{Name: "delta_client", Function: newDelta(ctx, serverAddress)},
+			{Name: "delta_client", Function: newDelta(ctx, lakeFSAddr)},
 		})
 		return 1
 	}

From d9cde8374c7ac1042329e6cab847c0d74990f348 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:33:24 +0200
Subject: [PATCH 34/35] change delta client builder API and drop storage type
 (since always using the S3 gateway). Use regex to validate lakeFS address

---
 examples/hooks/delta_lake_S3_export.lua       |  3 ++-
 pkg/actions/lua/formats/delta.go              | 20 ++++++-------------
 .../lakefs/catalogexport/delta_exporter.lua   |  5 -----
 3 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/examples/hooks/delta_lake_S3_export.lua b/examples/hooks/delta_lake_S3_export.lua
index 6899be15795..2da5ad134c6 100644
--- a/examples/hooks/delta_lake_S3_export.lua
+++ b/examples/hooks/delta_lake_S3_export.lua
@@ -13,11 +13,12 @@
         - put_object: function(bucket, key, data)
 ]]
 local aws = require("aws")
+local formats = require("formats")
 local delta_export = require("lakefs/catalogexport/delta_exporter")
 
 local sc = aws.s3_client(args.aws.aws_access_key_id, args.aws.aws_secret_access_key, args.aws.aws_region)
 
-local delta_client = delta_export.get_delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.aws_region)
+local delta_client = formats.delta_client(args.lakefs.access_key_id, args.lakefs.secret_access_key, args.aws.aws_region)
 local delta_table_locations = delta_export.export_delta_log(action, args.table_paths, sc.put_object, delta_client)
 for t, loc in pairs(delta_table_locations) do
     print("Delta Lake exported table \"" .. t .. "\"'s location: " .. loc .. "\n")
diff --git a/pkg/actions/lua/formats/delta.go b/pkg/actions/lua/formats/delta.go
index f4fa98b6a96..ad297b8d161 100644
--- a/pkg/actions/lua/formats/delta.go
+++ b/pkg/actions/lua/formats/delta.go
@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"strings"
+	"regexp"
 
 	"github.com/Shopify/go-lua"
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -125,21 +125,13 @@ func (awsI AWSInfo) GetAccessProperties() (interface{}, error) {
 // newDelta is a factory function to create server/cloud specific Delta Lake client
 // lakeFSAddr is the domain or "authority:port" of the running lakeFS server
 func newDelta(ctx context.Context, lakeFSAddr string) lua.Function {
-	if strings.HasPrefix(lakeFSAddr, ":") {
+	if regexp.MustCompile(`^:\d+`).MatchString(lakeFSAddr) {
 		// workaround in case we listen on all interfaces without specifying ip
 		lakeFSAddr = fmt.Sprintf("localhost%s", lakeFSAddr)
 	}
 	lakeFSAddr = fmt.Sprintf("http://%s", lakeFSAddr)
 	return func(l *lua.State) int {
-		var client *DeltaClient
-		st := lua.CheckString(l, 1)
-		switch storageType(st) {
-		case s3StorageType:
-			client = newS3DeltaClient(l, ctx, lakeFSAddr)
-		default:
-			lua.Errorf(l, "unimplemented storage type")
-			panic("unimplemented storage type")
-		}
+		client := newS3DeltaClient(l, ctx, lakeFSAddr)
 		l.NewTable()
 		for name, goFn := range functions {
 			l.PushGoFunction(goFn(client))
@@ -150,9 +142,9 @@ func newDelta(ctx context.Context, lakeFSAddr string) lua.Function {
 }
 
 func newS3DeltaClient(l *lua.State, ctx context.Context, lakeFSAddr string) *DeltaClient {
-	accessKeyID := lua.CheckString(l, 2)
-	secretAccessKey := lua.CheckString(l, 3)
-	r := lua.CheckString(l, 4)
+	accessKeyID := lua.CheckString(l, 1)
+	secretAccessKey := lua.CheckString(l, 2)
+	r := lua.CheckString(l, 3)
 	awsProps := storage.AWSProperties{
 		Region:         r,
 		ForcePathStyle: true,
diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
index c37b6139f84..3525f7de2bc 100644
--- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
+++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua
@@ -28,10 +28,6 @@ local function delta_log_entry_key_generator()
     end
 end
 
-local function get_delta_client(key, secret, region)
-    return formats.delta_client("s3", key, secret, region)
-end
-
 --[[
     action:
         - repository_id
@@ -137,5 +133,4 @@ end
 
 return {
     export_delta_log = export_delta_log,
-    get_delta_client = get_delta_client
 }

From dd4c81dda1c97bcaea1237dc393807c09e538ec2 Mon Sep 17 00:00:00 2001
From: Jonathan Rosenberg
 <96974219+Jonathan-Rosenberg@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:37:11 +0200
Subject: [PATCH 35/35] rename the lakeFS address field and add clarifying
 comment

---
 pkg/actions/lua.go        | 2 +-
 pkg/actions/lua/open.go   | 2 +-
 pkg/actions/lua/stdlib.go | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/actions/lua.go b/pkg/actions/lua.go
index a9abbad7b36..aec049d78fb 100644
--- a/pkg/actions/lua.go
+++ b/pkg/actions/lua.go
@@ -92,7 +92,7 @@ func (h *LuaHook) Run(ctx context.Context, record graveler.HookRecord, buf *byte
 	l := lua.NewState()
 	osc := lualibs.OpenSafeConfig{
 		NetHTTPEnabled: h.Config.Lua.NetHTTPEnabled,
-		ServerAddress:  h.serverAddress,
+		LakeFSAddr:     h.serverAddress,
 	}
 	lualibs.OpenSafe(l, ctx, osc, &loggingBuffer{buf: buf, ctx: ctx})
 	injectHookContext(l, ctx, user, h.Endpoint, h.Args)
diff --git a/pkg/actions/lua/open.go b/pkg/actions/lua/open.go
index c0eef50ddaa..3bf4fb11ff3 100644
--- a/pkg/actions/lua/open.go
+++ b/pkg/actions/lua/open.go
@@ -46,7 +46,7 @@ func Open(l *lua.State, ctx context.Context, cfg OpenSafeConfig) {
 	aws.Open(l, ctx)
 	gcloud.Open(l, ctx)
 	url.Open(l)
-	formats.Open(l, ctx, cfg.ServerAddress)
+	formats.Open(l, ctx, cfg.LakeFSAddr)
 	if cfg.NetHTTPEnabled {
 		http.Open(l)
 	}
diff --git a/pkg/actions/lua/stdlib.go b/pkg/actions/lua/stdlib.go
index f420fa48f3d..ae929c3f93a 100644
--- a/pkg/actions/lua/stdlib.go
+++ b/pkg/actions/lua/stdlib.go
@@ -350,7 +350,7 @@ func BaseOpen(buf io.StringWriter) glua.Function {
 
 type OpenSafeConfig struct {
 	NetHTTPEnabled bool
-	ServerAddress  string
+	LakeFSAddr     string // The domain (or "authority:port") that lakeFS listens to
 }
 
 func OpenSafe(l *glua.State, ctx context.Context, cfg OpenSafeConfig, buf io.StringWriter) {