1705 lancedb orphan removal via staging deletion inserts

dlt/destinations/impl/lancedb/configuration.py

@@ -59,6 +59,7 @@ class LanceDBClientOptions(BaseConfiguration):
     "sentence-transformers",
     "huggingface",
     "colbert",
+    "ollama",
 ]
 
 

dlt/destinations/impl/lancedb/factory.py

@@ -16,8 +16,8 @@ class lancedb(Destination[LanceDBClientConfiguration, "LanceDBClient"]):
 
     def _raw_capabilities(self) -> DestinationCapabilitiesContext:
         caps = DestinationCapabilitiesContext()
-        caps.preferred_loader_file_format = "jsonl"
-        caps.supported_loader_file_formats = ["jsonl"]
+        caps.preferred_loader_file_format = "parquet"
+        caps.supported_loader_file_formats = ["parquet", "reference"]
 
         caps.max_identifier_length = 200
         caps.max_column_identifier_length = 1024
@@ -30,6 +30,8 @@ def _raw_capabilities(self) -> DestinationCapabilitiesContext:
         caps.decimal_precision = (38, 18)
         caps.timestamp_precision = 6
 
+        caps.recommended_file_size = 128_000_000
+
         return caps
 
     @property

dlt/destinations/impl/lancedb/lancedb_adapter.py

@@ -6,11 +6,13 @@
 
 
 VECTORIZE_HINT = "x-lancedb-embed"
+DOCUMENT_ID_HINT = "x-lancedb-doc-id"
 
 
 def lancedb_adapter(
     data: Any,
     embed: TColumnNames = None,
+    document_id: TColumnNames = None,
 ) -> DltResource:
     """Prepares data for the LanceDB destination by specifying which columns should be embedded.
 
@@ -20,6 +22,8 @@ def lancedb_adapter(
             object.
         embed (TColumnNames, optional): Specify columns to generate embeddings for.
             It can be a single column name as a string, or a list of column names.
+        document_id (TColumnNames, optional): Specify columns which represenet the document
+            and which will be appended to primary/merge keys.
 
     Returns:
         DltResource: A resource with applied LanceDB-specific hints.
@@ -50,8 +54,22 @@ def lancedb_adapter(
                 VECTORIZE_HINT: True,  # type: ignore[misc]
             }
 
+    if document_id:
+        if isinstance(document_id, str):
+            document_id = [document_id]
+        if not isinstance(document_id, list):
+            raise ValueError(
+                "'document_id' must be a list of column names or a single column name as a string."
+            )
+
+        for column_name in document_id:
+            column_hints[column_name] = {
+                "name": column_name,
+                DOCUMENT_ID_HINT: True,  # type: ignore[misc]
+            }
+
     if not column_hints:
-        raise ValueError("A value for 'embed' must be specified.")
+        raise ValueError("At least one of 'embed' or 'document_id' must be specified.")
     else:
         resource.apply_hints(columns=column_hints)