supports include null cursor_path rows for RecordBatches

dlt-hub · Aug 7, 2024 · d4b8b68 · d4b8b68
1 parent 6d7d4c0
commit d4b8b68
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/dlt/extract/incremental/exceptions.py b/dlt/extract/incremental/exceptions.py
@@ -2,8 +2,6 @@
 from dlt.common.typing import TDataItem
 
 
-
-
 class IncrementalCursorPathMissing(PipeException):
     def __init__(
         self, pipe_name: str, json_path: str, item: TDataItem = None, msg: str = None

diff --git a/dlt/extract/incremental/transform.py b/dlt/extract/incremental/transform.py
@@ -386,7 +386,11 @@ def __call__(
                 )
             )
         if self.on_cursor_value_missing == "include":
-            tbl = pa.concat_tables([tbl, tbl_with_null])
+            if isinstance(tbl, pa.RecordBatch):
+                assert isinstance(tbl_with_null, pa.RecordBatch)
+                tbl = pa.Table.from_batches([tbl, tbl_with_null])
+            else:
+                tbl = pa.concat_tables([tbl, tbl_with_null])
 
         if len(tbl) == 0:
             return None, start_out_of_range, end_out_of_range