From 178107dc7be9700dc017ffd85e5dc4406031efef Mon Sep 17 00:00:00 2001 From: Jonathan Rosenberg <96974219+Jonathan-Rosenberg@users.noreply.github.com> Date: Sun, 3 Dec 2023 15:05:03 +0200 Subject: [PATCH] comment the delta_log_entry_key_generator function and return a complete key name (including ".json" suffix) --- .../lua/lakefs/catalogexport/delta_exporter.lua | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua index aa378c6ba96..4c9fadd1e72 100644 --- a/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua +++ b/pkg/actions/lua/lakefs/catalogexport/delta_exporter.lua @@ -4,6 +4,14 @@ local pathlib = require("path") local json = require("encoding/json") local utils = require("lakefs/catalogexport/internal") +--[[ + delta_log_entry_key_generator returns a closure that returns a Delta Lake version key according to the Delta Lake + protocol: https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries + Example: + local gen = delta_log_entry_key_generator() + gen() -- 000000000000000001.json + gen() -- 000000000000000002.json +]] local function delta_log_entry_key_generator() local current = 0 return function() @@ -14,7 +22,7 @@ local function delta_log_entry_key_generator() for _ = 1, padding_length do padded_key = padded_key .. "0" end - padded_key = padded_key .. key + padded_key = padded_key .. key .. ".json" current = current + 1 return padded_key end @@ -93,7 +101,7 @@ local function export_delta_log(action, table_paths, storage_client, delta) local entry_m = json.marshal(entry) table.insert(entry_log, entry_m) end - table_log[string.format("%s.json", keyGenerator())] = entry_log + table_log[keyGenerator()] = entry_log end -- Get the table delta log physical location