Enable schema evolution for merge write disposition with delta ta…

…ble format (#1742)

* black format

* increase minimum deltalake version dependency

* enable schema evolution for delta table merge

* extract delta table merge logic into separate function

* remove big decimal exclusion due to upstream bugfix

* evolve delta table schema in empty source case

* refactor DeltaLoadFilesystemJob

* uses right table path format in delta lake load job

* allows to pass schema name when getting delta tables and computing table counts

* cleansup usage of remote paths and uris in filesystem load jobs

* removes tempfile from file_storage

---------

Co-authored-by: Marcin Rudolf <[email protected]>

pyproject.toml

@@ -80,7 +80,7 @@ databricks-sql-connector = {version = ">=2.9.3", optional = true}
 clickhouse-driver = { version = ">=0.2.7", optional = true }
 clickhouse-connect = { version = ">=0.7.7", optional = true }
 lancedb = { version = ">=0.8.2", optional = true, markers = "python_version >= '3.9'", allow-prereleases = true }
-deltalake = { version = ">=0.17.4", optional = true }
+deltalake = { version = ">=0.19.0", optional = true }
 graphlib-backport = {version = "*", python = "<3.9"}
 
 [tool.poetry.extras]