From 43f79ad6153c912d16f2ffb724016677c52a821a Mon Sep 17 00:00:00 2001 From: Jorrit Sandbrink <47451109+jorritsandbrink@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:59:43 +0400 Subject: [PATCH] Enable schema evolution for `merge` write disposition with `delta` table format (#1742) * black format * increase minimum deltalake version dependency * enable schema evolution for delta table merge * extract delta table merge logic into separate function * remove big decimal exclusion due to upstream bugfix * evolve delta table schema in empty source case * refactor DeltaLoadFilesystemJob * uses right table path format in delta lake load job * allows to pass schema name when getting delta tables and computing table counts * cleansup usage of remote paths and uris in filesystem load jobs * removes tempfile from file_storage --------- Co-authored-by: Marcin Rudolf --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a33168acd2..2e7c7a971f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ databricks-sql-connector = {version = ">=2.9.3", optional = true} clickhouse-driver = { version = ">=0.2.7", optional = true } clickhouse-connect = { version = ">=0.7.7", optional = true } lancedb = { version = ">=0.8.2", optional = true, markers = "python_version >= '3.9'", allow-prereleases = true } -deltalake = { version = ">=0.17.4", optional = true } +deltalake = { version = ">=0.19.0", optional = true } graphlib-backport = {version = "*", python = "<3.9"} [tool.poetry.extras]