From c9f5fb5e6832363d51d0c37e8c052992334e0cc5 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 16:03:47 +0200 Subject: [PATCH 01/13] init implementation --- dbt/adapters/athena/__version__.py | 2 +- .../models/helpers_iceberg.sql | 74 +++++++++++++++++++ .../materializations/models/table/table.sql | 32 ++++++-- 3 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 dbt/include/athena/macros/materializations/models/helpers_iceberg.sql diff --git a/dbt/adapters/athena/__version__.py b/dbt/adapters/athena/__version__.py index 86774adc..40986530 100644 --- a/dbt/adapters/athena/__version__.py +++ b/dbt/adapters/athena/__version__.py @@ -1 +1 @@ -version = "1.0.1" +version = "1.0.2" diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql new file mode 100644 index 00000000..9985f3f2 --- /dev/null +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -0,0 +1,74 @@ +{% macro drop_iceberg(relation) -%} + drop table if exists {{ relation }} +{% endmacro %} + +{% macro create_table_iceberg(relation, tmp_relation, sql) -%} + {%- set external_location = config.get('external_location', default=none) -%} + {%- set staging_location = config.get('staging_location') -%} + {%- set partitioned_by = config.get('partitioned_by', default=none) -%} + {%- set bucketed_by = config.get('bucketed_by', default=none) -%} + {%- set bucket_count = config.get('bucket_count', default=none) -%} + {%- set write_compression = config.get('write_compression', default=none) -%} + + {%- set target_relation = this.incorporate(type='table') -%} + + {% if tmp_relation is not none %} + {% do adapter.drop_relation(tmp_relation) %} + {% endif %} + + -- create tmp table + + {% do run_query(create_tmp_table_iceberg(tmp_relation, sql, staging_location)) %} + + {%- set dest_columns = adapter.get_columns_in_relation(tmp_relation) -%} + + {% do run_query(create_iceberg_table_definition(target_relation, dest_columns)) %} + + {{ return(incremental_insert(tmp_relation, target_relation)) }} + +{% endmacro %} + + +{% macro create_tmp_table_iceberg(relation, sql, staging_location) -%} + create table + {{ relation }} + + with ( + external_location='{{ staging_location }}', + write_compression='snappy', + format='parquet' + ) + as + {{ sql }} +{% endmacro %} + +{% macro create_iceberg_table_definition(relation, dest_columns) -%} + {%- set external_location = config.get('external_location', default=none) -%} + {%- set partitioned_by = config.get('partitioned_by', default=none) -%} + {%- set partitioned_by_csv = partitioned_by | join(', ') -%} + {%- set dest_columns_with_type = [] -%} + + {%- for col in dest_columns -%} + {%- if 'varchar' in col.dtype -%} + {% set dtype = 'string' -%} + {%- else -%} + {% set dtype = col.dtype -%} + {%- endif -%} + + {% set t = dest_columns_with_type.append(col.name + ' ' + dtype) -%} + {%- endfor -%} + + {%- set dest_columns_with_type_csv = dest_columns_with_type | join(', ') -%} + + + CREATE TABLE {{ relation }} ( + {{ dest_columns_with_type_csv }} + ) + {%- if partitioned_by is not none %} + PARTITIONED BY ({{partitioned_by_csv}}) + {%- endif %} + LOCATION '{{ external_location }}' + TBLPROPERTIES ( 'table_type' = 'ICEBERG' ) + +{% endmacro %} + diff --git a/dbt/include/athena/macros/materializations/models/table/table.sql b/dbt/include/athena/macros/materializations/models/table/table.sql index ad425f00..858861eb 100644 --- a/dbt/include/athena/macros/materializations/models/table/table.sql +++ b/dbt/include/athena/macros/materializations/models/table/table.sql @@ -1,6 +1,6 @@ {% materialization table, adapter='athena' -%} {%- set identifier = model['alias'] -%} - + {%- set format = config.get('format', default='parquet') -%} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, @@ -10,16 +10,36 @@ {{ run_hooks(pre_hooks) }} {%- if old_relation is not none -%} - {{ adapter.drop_relation(old_relation) }} + {%- if format == 'iceberg' -%} + {% do run_query(drop_iceberg(old_relation)) %} + {% else %} + {{ adapter.drop_relation(old_relation) }} + {%- endif -%} {%- endif -%} -- build model - {% call statement('main') -%} - {{ create_table_as(False, target_relation, sql) }} - {% endcall -%} + {%- if format == 'iceberg' -%} + {%- set tmp_relation = make_temp_relation(target_relation) -%} + {%- set build_sql = create_table_iceberg(target_relation, tmp_relation, sql) -%} + {% else %} + {% set build_sql = create_table_as(False, target_relation, sql) -%} + {%- endif -%} + + {% call statement("main") %} + {{ build_sql }} + {% endcall %} + + + -- drop tmp table in case of iceberg + {%- if format == 'iceberg' -%} + {% do adapter.drop_relation(tmp_relation) %} + {%- endif -%} + -- set table properties - {{ set_table_classification(target_relation, 'parquet') }} + {%- if format == 'parquet' -%} + {{ set_table_classification(target_relation, 'parquet') }} + {%- endif -%} {{ run_hooks(post_hooks) }} From 65e3d0050c3545537aa7fcf67905bf72efae48d2 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 16:26:06 +0200 Subject: [PATCH 02/13] add default location --- .../macros/materializations/models/helpers_iceberg.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index 9985f3f2..fb3e01e8 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -34,7 +34,6 @@ {{ relation }} with ( - external_location='{{ staging_location }}', write_compression='snappy', format='parquet' ) @@ -48,6 +47,11 @@ {%- set partitioned_by_csv = partitioned_by | join(', ') -%} {%- set dest_columns_with_type = [] -%} + {%- if external_location is none %} + {%- set default_location = target.s3_staging_dir -%} + {%- set external_location= default_location + relation.name + '/' -%} + {%- endif %} + {%- for col in dest_columns -%} {%- if 'varchar' in col.dtype -%} {% set dtype = 'string' -%} From 00ce6e07d57cf9a65fb0984b76f238b01e17f450 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 16:27:58 +0200 Subject: [PATCH 03/13] example --- .../athena/macros/materializations/models/table/table.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/table/table.sql b/dbt/include/athena/macros/materializations/models/table/table.sql index 858861eb..115b8ec6 100644 --- a/dbt/include/athena/macros/materializations/models/table/table.sql +++ b/dbt/include/athena/macros/materializations/models/table/table.sql @@ -29,7 +29,6 @@ {{ build_sql }} {% endcall %} - -- drop tmp table in case of iceberg {%- if format == 'iceberg' -%} {% do adapter.drop_relation(tmp_relation) %} @@ -37,8 +36,8 @@ -- set table properties - {%- if format == 'parquet' -%} - {{ set_table_classification(target_relation, 'parquet') }} + {%- if format != 'iceberg' -%} + {{ set_table_classification(target_relation, format) }} {%- endif -%} {{ run_hooks(post_hooks) }} From 479cb0df48f07d13f2e2026191e56e1afceb86f8 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 16:32:11 +0200 Subject: [PATCH 04/13] revert set_table_classification --- .../athena/macros/materializations/models/table/table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/athena/macros/materializations/models/table/table.sql b/dbt/include/athena/macros/materializations/models/table/table.sql index 115b8ec6..e024fca9 100644 --- a/dbt/include/athena/macros/materializations/models/table/table.sql +++ b/dbt/include/athena/macros/materializations/models/table/table.sql @@ -37,7 +37,7 @@ -- set table properties {%- if format != 'iceberg' -%} - {{ set_table_classification(target_relation, format) }} + {{ set_table_classification(target_relation, 'parquet') }} {%- endif -%} {{ run_hooks(post_hooks) }} From db878caa4df115bb2d02c2735953f052d2601ecd Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 16:37:41 +0200 Subject: [PATCH 05/13] fix format --- .../athena/macros/materializations/models/table/table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/athena/macros/materializations/models/table/table.sql b/dbt/include/athena/macros/materializations/models/table/table.sql index e024fca9..115b8ec6 100644 --- a/dbt/include/athena/macros/materializations/models/table/table.sql +++ b/dbt/include/athena/macros/materializations/models/table/table.sql @@ -37,7 +37,7 @@ -- set table properties {%- if format != 'iceberg' -%} - {{ set_table_classification(target_relation, 'parquet') }} + {{ set_table_classification(target_relation, format) }} {%- endif -%} {{ run_hooks(post_hooks) }} From 6937e49b91114d3ee8fc324688a70d74b7d17ef4 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 18:43:38 +0200 Subject: [PATCH 06/13] add support for integer --- .../athena/macros/materializations/models/helpers_iceberg.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index fb3e01e8..c5df4dce 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -55,6 +55,8 @@ {%- for col in dest_columns -%} {%- if 'varchar' in col.dtype -%} {% set dtype = 'string' -%} + {%- elif 'integer' == col.dtype -%} + {% set dtype = 'int' -%} {%- else -%} {% set dtype = col.dtype -%} {%- endif -%} From 7997b6efaed471852a76a01935062c715d5a2bf6 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 19:22:38 +0200 Subject: [PATCH 07/13] refactor data_types --- .../models/helpers_iceberg.sql | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index c5df4dce..755ddf6d 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -53,14 +53,7 @@ {%- endif %} {%- for col in dest_columns -%} - {%- if 'varchar' in col.dtype -%} - {% set dtype = 'string' -%} - {%- elif 'integer' == col.dtype -%} - {% set dtype = 'int' -%} - {%- else -%} - {% set dtype = col.dtype -%} - {%- endif -%} - + {% set dtype = iceberg_data_type(col.dtype) -%} {% set t = dest_columns_with_type.append(col.name + ' ' + dtype) -%} {%- endfor -%} @@ -78,3 +71,15 @@ {% endmacro %} + +{% macro iceberg_data_type(col_type) -%} + {%- if 'varchar' in col_type -%} + {% set data_type = 'string' -%} + {%- elif 'integer' == col_type -%} + {% set data_type = 'int' -%} + {%- else -%} + {% set data_type = col_type -%} + {%- endif -%} + + {{ return(data_type) }} +{% endmacro %} From 226db7ba25ada7aa85f3ecb272b69f7d4acbc564 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 19:29:08 +0200 Subject: [PATCH 08/13] some tweaks --- .../materializations/models/helpers_iceberg.sql | 11 +++++++++-- .../macros/materializations/models/table/table.sql | 8 +++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index 755ddf6d..f0827d61 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -2,7 +2,7 @@ drop table if exists {{ relation }} {% endmacro %} -{% macro create_table_iceberg(relation, tmp_relation, sql) -%} +{% macro create_table_iceberg(relation, old_relation, tmp_relation, sql) -%} {%- set external_location = config.get('external_location', default=none) -%} {%- set staging_location = config.get('staging_location') -%} {%- set partitioned_by = config.get('partitioned_by', default=none) -%} @@ -17,13 +17,20 @@ {% endif %} -- create tmp table - {% do run_query(create_tmp_table_iceberg(tmp_relation, sql, staging_location)) %} + -- get columns from tmp table to support metadata creation {%- set dest_columns = adapter.get_columns_in_relation(tmp_relation) -%} + -- drop old relation only if tmp table is ready + {%- if old_relation is not none -%} + {% do run_query(drop_iceberg(old_relation)) %} + {%- endif -%} + + -- create iceberg table {% do run_query(create_iceberg_table_definition(target_relation, dest_columns)) %} + -- return final insert statement {{ return(incremental_insert(tmp_relation, target_relation)) }} {% endmacro %} diff --git a/dbt/include/athena/macros/materializations/models/table/table.sql b/dbt/include/athena/macros/materializations/models/table/table.sql index 115b8ec6..5cfbf5e4 100644 --- a/dbt/include/athena/macros/materializations/models/table/table.sql +++ b/dbt/include/athena/macros/materializations/models/table/table.sql @@ -10,17 +10,15 @@ {{ run_hooks(pre_hooks) }} {%- if old_relation is not none -%} - {%- if format == 'iceberg' -%} - {% do run_query(drop_iceberg(old_relation)) %} - {% else %} - {{ adapter.drop_relation(old_relation) }} + {%- if format != 'iceberg' -%} + {{ adapter.drop_relation(old_relation) }} {%- endif -%} {%- endif -%} -- build model {%- if format == 'iceberg' -%} {%- set tmp_relation = make_temp_relation(target_relation) -%} - {%- set build_sql = create_table_iceberg(target_relation, tmp_relation, sql) -%} + {%- set build_sql = create_table_iceberg(target_relation, old_relation, tmp_relation, sql) -%} {% else %} {% set build_sql = create_table_as(False, target_relation, sql) -%} {%- endif -%} From 6aad799db13c6da6fa30208d7686dbc8a2d6ce8a Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Mon, 17 Oct 2022 23:07:10 +0200 Subject: [PATCH 09/13] ad table properties and tweak partition --- .../models/helpers_iceberg.sql | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index f0827d61..49483aee 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -51,30 +51,40 @@ {% macro create_iceberg_table_definition(relation, dest_columns) -%} {%- set external_location = config.get('external_location', default=none) -%} {%- set partitioned_by = config.get('partitioned_by', default=none) -%} - {%- set partitioned_by_csv = partitioned_by | join(', ') -%} + {%- set table_properties = config.get('table_properties', default={}) -%} + {%- set _ = table_properties.update({'table_type': 'ICEBERG'}) -%} + {%- set table_properties_formatted = [] -%} {%- set dest_columns_with_type = [] -%} + {%- for k in table_properties -%} + {% set _ = table_properties_formatted.append("'" + k + "'='" + table_properties[k] + "'") -%} + {%- endfor -%} + + {%- set table_properties_csv= table_properties_formatted | join(', ') -%} + {%- if external_location is none %} - {%- set default_location = target.s3_staging_dir -%} - {%- set external_location= default_location + relation.name + '/' -%} + {%- set default_location = target.s3_staging_dir -%} + {%- set external_location= default_location + relation.name + '/' -%} {%- endif %} {%- for col in dest_columns -%} {% set dtype = iceberg_data_type(col.dtype) -%} - {% set t = dest_columns_with_type.append(col.name + ' ' + dtype) -%} + {% set _ = dest_columns_with_type.append(col.name + ' ' + dtype) -%} {%- endfor -%} {%- set dest_columns_with_type_csv = dest_columns_with_type | join(', ') -%} - CREATE TABLE {{ relation }} ( {{ dest_columns_with_type_csv }} ) {%- if partitioned_by is not none %} - PARTITIONED BY ({{partitioned_by_csv}}) + {%- set partitioned_by_csv = partitioned_by | join(', ') -%} + PARTITIONED BY ({{partitioned_by_csv}}) {%- endif %} LOCATION '{{ external_location }}' - TBLPROPERTIES ( 'table_type' = 'ICEBERG' ) + TBLPROPERTIES ( + {{table_properties_csv}} + ) {% endmacro %} From 151100805dfe3502db23f29e7f7185b198b38048 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Tue, 18 Oct 2022 07:32:11 +0200 Subject: [PATCH 10/13] tweaks some docs --- .../macros/materializations/models/helpers_iceberg.sql | 5 ++--- .../athena/macros/materializations/models/table/table.sql | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index 49483aee..4315d723 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -19,10 +19,10 @@ -- create tmp table {% do run_query(create_tmp_table_iceberg(tmp_relation, sql, staging_location)) %} - -- get columns from tmp table to support metadata creation + -- get columns from tmp table to retrieve metadata {%- set dest_columns = adapter.get_columns_in_relation(tmp_relation) -%} - -- drop old relation only if tmp table is ready + -- drop old relation after tmp table is ready {%- if old_relation is not none -%} {% do run_query(drop_iceberg(old_relation)) %} {%- endif -%} @@ -88,7 +88,6 @@ {% endmacro %} - {% macro iceberg_data_type(col_type) -%} {%- if 'varchar' in col_type -%} {% set data_type = 'string' -%} diff --git a/dbt/include/athena/macros/materializations/models/table/table.sql b/dbt/include/athena/macros/materializations/models/table/table.sql index 5cfbf5e4..2c8bd9f0 100644 --- a/dbt/include/athena/macros/materializations/models/table/table.sql +++ b/dbt/include/athena/macros/materializations/models/table/table.sql @@ -9,6 +9,7 @@ {{ run_hooks(pre_hooks) }} + -- drop old relation if it is not iceberg {%- if old_relation is not none -%} {%- if format != 'iceberg' -%} {{ adapter.drop_relation(old_relation) }} @@ -32,7 +33,6 @@ {% do adapter.drop_relation(tmp_relation) %} {%- endif -%} - -- set table properties {%- if format != 'iceberg' -%} {{ set_table_classification(target_relation, format) }} From b1975c9eebfd63b767c1dcc6f8e7a652f09a0865 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:10:29 +0200 Subject: [PATCH 11/13] improve external location do add uniqueness --- dbt/adapters/athena/impl.py | 16 ++++++++++++++++ .../materializations/models/helpers_iceberg.sql | 5 +---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index 165c77b2..44664e47 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -3,6 +3,7 @@ import boto3 from botocore.exceptions import ClientError from itertools import chain +from os import path from threading import Lock from typing import Dict, Iterator, List, Optional, Set from uuid import uuid4 @@ -52,6 +53,21 @@ def s3_uuid_table_location(self): return f"{client.s3_staging_dir}tables/{str(uuid4())}/" + @available + def get_unique_external_location(self, external_location, staging_dir, relation_name): + """ + Generate a unique not overlapping location. + """ + unique_id = str(uuid4()) + if external_location is not None: + if external_location.endswith('/'): + external_location = external_location[:-1] + external_location = f'{external_location}_{unique_id}/' + else: + base_path = path.join(staging_dir, f'{relation_name}_{unique_id}') + external_location = f'{base_path}/' + return external_location + @available def clean_up_partitions( self, database_name: str, table_name: str, where_condition: str diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index 4315d723..98ca9400 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -62,10 +62,7 @@ {%- set table_properties_csv= table_properties_formatted | join(', ') -%} - {%- if external_location is none %} - {%- set default_location = target.s3_staging_dir -%} - {%- set external_location= default_location + relation.name + '/' -%} - {%- endif %} + {%- set external_location = adapter.get_unique_external_location(external_location, target.s3_staging_dir, relation.name) -%} {%- for col in dest_columns -%} {% set dtype = iceberg_data_type(col.dtype) -%} From 499a71eff3c7ab5fda722255209f66f39f243f96 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:56:17 +0200 Subject: [PATCH 12/13] add strict location param --- dbt/adapters/athena/impl.py | 9 +++++---- .../macros/materializations/models/helpers_iceberg.sql | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/athena/impl.py b/dbt/adapters/athena/impl.py index 44664e47..c1eb5cd3 100755 --- a/dbt/adapters/athena/impl.py +++ b/dbt/adapters/athena/impl.py @@ -54,15 +54,16 @@ def s3_uuid_table_location(self): return f"{client.s3_staging_dir}tables/{str(uuid4())}/" @available - def get_unique_external_location(self, external_location, staging_dir, relation_name): + def get_unique_external_location(self, external_location, strict_location, staging_dir, relation_name): """ Generate a unique not overlapping location. """ unique_id = str(uuid4()) if external_location is not None: - if external_location.endswith('/'): - external_location = external_location[:-1] - external_location = f'{external_location}_{unique_id}/' + if not strict_location: + if external_location.endswith('/'): + external_location = external_location[:-1] + external_location = f'{external_location}_{unique_id}/' else: base_path = path.join(staging_dir, f'{relation_name}_{unique_id}') external_location = f'{base_path}/' diff --git a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql index 98ca9400..9aeb549c 100644 --- a/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql +++ b/dbt/include/athena/macros/materializations/models/helpers_iceberg.sql @@ -50,6 +50,7 @@ {% macro create_iceberg_table_definition(relation, dest_columns) -%} {%- set external_location = config.get('external_location', default=none) -%} + {%- set strict_location = config.get('strict_location', default=true) -%} {%- set partitioned_by = config.get('partitioned_by', default=none) -%} {%- set table_properties = config.get('table_properties', default={}) -%} {%- set _ = table_properties.update({'table_type': 'ICEBERG'}) -%} @@ -62,7 +63,7 @@ {%- set table_properties_csv= table_properties_formatted | join(', ') -%} - {%- set external_location = adapter.get_unique_external_location(external_location, target.s3_staging_dir, relation.name) -%} + {%- set external_location = adapter.get_unique_external_location(external_location, strict_location, target.s3_staging_dir, relation.name) -%} {%- for col in dest_columns -%} {% set dtype = iceberg_data_type(col.dtype) -%} From 2be6eed083866cdf2767392a70765269687e06b3 Mon Sep 17 00:00:00 2001 From: nicor88 <6278547+nicor88@users.noreply.github.com> Date: Wed, 9 Nov 2022 16:08:49 +0100 Subject: [PATCH 13/13] Delete __version__.py --- dbt/adapters/athena/__version__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 dbt/adapters/athena/__version__.py diff --git a/dbt/adapters/athena/__version__.py b/dbt/adapters/athena/__version__.py deleted file mode 100644 index 40986530..00000000 --- a/dbt/adapters/athena/__version__.py +++ /dev/null @@ -1 +0,0 @@ -version = "1.0.2"