From b71eb57712584dd906e99793d7507154f7acdf3c Mon Sep 17 00:00:00 2001 From: dave Date: Wed, 11 Dec 2024 11:56:38 +0100 Subject: [PATCH 1/4] try to fix ibis az problems on linux --- dlt/helpers/ibis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dlt/helpers/ibis.py b/dlt/helpers/ibis.py index ed4264dac7..5376fe6a1c 100644 --- a/dlt/helpers/ibis.py +++ b/dlt/helpers/ibis.py @@ -123,8 +123,11 @@ def create_ibis_backend( ) from dlt.destinations.impl.duckdb.factory import DuckDbCredentials - # we create an in memory duckdb and create all tables on there + # we create an in memory duckdb and create the ibis backend from it duck = duckdb.connect(":memory:") + con = ibis.duckdb.from_connection(duck) + + # make all tables availble here fs_client = cast(FilesystemClient, client) creds = DuckDbCredentials(duck) sql_client = FilesystemSqlClient( @@ -134,7 +137,6 @@ def create_ibis_backend( # NOTE: we should probably have the option for the user to only select a subset of tables here with sql_client as _: sql_client.create_views_for_all_tables() - con = ibis.duckdb.from_connection(duck) return con From 476487ab1b2b94470016ce059d5a2a82cf51a912 Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 11 Dec 2024 23:14:25 +0100 Subject: [PATCH 2/4] remove duckdb certs fix --- .github/workflows/test_destinations.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 84a8f95d71..a9306c2f9c 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -79,9 +79,6 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E redshift -E postgis -E postgres -E gs -E s3 -E az -E parquet -E duckdb -E cli -E filesystem --with sentry-sdk --with pipeline,ibis -E deltalake -E pyiceberg - - name: enable certificates for azure and duckdb - run: sudo mkdir -p /etc/pki/tls/certs && sudo ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt - - name: Upgrade sqlalchemy run: poetry run pip install sqlalchemy==2.0.18 # minimum version required by `pyiceberg` From 0a666d12091131da50e10e0bd9294a8d2193cc3c Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 12 Dec 2024 08:04:18 +0100 Subject: [PATCH 3/4] test explicitely setting transport options --- dlt/helpers/ibis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dlt/helpers/ibis.py b/dlt/helpers/ibis.py index 5376fe6a1c..6cb8dd5173 100644 --- a/dlt/helpers/ibis.py +++ b/dlt/helpers/ibis.py @@ -137,6 +137,8 @@ def create_ibis_backend( # NOTE: we should probably have the option for the user to only select a subset of tables here with sql_client as _: sql_client.create_views_for_all_tables() + sql_client._conn.sql("SET azure_transport_option_type = 'curl';") + return con From da975d69431a6354108c11af93821179c6f1e7f1 Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Sun, 15 Dec 2024 16:39:08 +0100 Subject: [PATCH 4/4] sets the ssl curl on a correct connection clone --- dlt/helpers/ibis.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dlt/helpers/ibis.py b/dlt/helpers/ibis.py index 6cb8dd5173..e15bb9bc16 100644 --- a/dlt/helpers/ibis.py +++ b/dlt/helpers/ibis.py @@ -10,7 +10,7 @@ import sqlglot from ibis import BaseBackend, Expr except ModuleNotFoundError: - raise MissingDependencyException("dlt ibis Helpers", ["ibis"]) + raise MissingDependencyException("dlt ibis helpers", ["ibis-framework"]) SUPPORTED_DESTINATIONS = [ @@ -124,21 +124,21 @@ def create_ibis_backend( from dlt.destinations.impl.duckdb.factory import DuckDbCredentials # we create an in memory duckdb and create the ibis backend from it - duck = duckdb.connect(":memory:") - con = ibis.duckdb.from_connection(duck) - - # make all tables availble here fs_client = cast(FilesystemClient, client) - creds = DuckDbCredentials(duck) sql_client = FilesystemSqlClient( - fs_client, dataset_name=fs_client.dataset_name, credentials=creds + fs_client, + dataset_name=fs_client.dataset_name, + credentials=DuckDbCredentials(duckdb.connect()), ) - + # do not use context manager to not return and close the cloned connection + duckdb_conn = sql_client.open_connection() + # make all tables available here # NOTE: we should probably have the option for the user to only select a subset of tables here - with sql_client as _: - sql_client.create_views_for_all_tables() - sql_client._conn.sql("SET azure_transport_option_type = 'curl';") - + sql_client.create_views_for_all_tables() + # why this works now: whenever a clone of connection is made, all SET commands + # apply only to it. old code was setting `curl` on the internal clone of sql_client + # now we export this clone directly to ibis to it works + con = ibis.duckdb.from_connection(duckdb_conn) return con