From bacbe2348358e66ee072d9966bab93e2889d9418 Mon Sep 17 00:00:00 2001 From: Rodrigo Carneiro Date: Wed, 20 Nov 2024 13:59:52 -0300 Subject: [PATCH 1/4] add unique temp table suffix for iceberg tables (#757) Co-authored-by: Carneiro, Rodrigo --- .../materializations/models/incremental/incremental.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql b/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql index b9f5994e..c18ac681 100644 --- a/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql +++ b/dbt-athena/src/dbt/include/athena/macros/materializations/models/incremental/incremental.sql @@ -20,6 +20,10 @@ {% set tmp_table_suffix = '__dbt_tmp' %} {% endif %} + {% if unique_tmp_table_suffix == True and table_type == 'iceberg' %} + {% set tmp_table_suffix = adapter.generate_unique_temporary_table_suffix() %} + {% endif %} + {% set old_tmp_relation = adapter.get_relation(identifier=target_relation.identifier ~ tmp_table_suffix, schema=schema, database=database) %} From a425cc6784ec89158f390178d08338fe4cbe4d34 Mon Sep 17 00:00:00 2001 From: Alexandro Hou Date: Sat, 23 Nov 2024 19:31:58 +0000 Subject: [PATCH 2/4] Invalid tmp table name when using: unique_tmp_table_suffix (#754) Co-authored-by: Alexandro Hou Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- dbt-athena/src/dbt/adapters/athena/impl.py | 9 ++++----- .../functional/adapter/test_unique_tmp_table_suffix.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dbt-athena/src/dbt/adapters/athena/impl.py b/dbt-athena/src/dbt/adapters/athena/impl.py index 5b94a4c9..e70cca3c 100755 --- a/dbt-athena/src/dbt/adapters/athena/impl.py +++ b/dbt-athena/src/dbt/adapters/athena/impl.py @@ -429,7 +429,7 @@ def clean_up_table(self, relation: AthenaRelation) -> None: @available def generate_unique_temporary_table_suffix(self, suffix_initial: str = "__dbt_tmp") -> str: - return f"{suffix_initial}_{str(uuid4())}" + return f"{suffix_initial}_{str(uuid4()).replace('-', '_')}" def quote(self, identifier: str) -> str: return f"{self.quote_character}{identifier}{self.quote_character}" @@ -1209,22 +1209,21 @@ def _generate_snapshot_migration_sql(self, relation: AthenaRelation, table_colum - Copy the content of the staging table to the final table - Delete the staging table """ - col_csv = f",\n{' ' * 16}".join(table_columns) + col_csv = f", \n{' ' * 16}".join(table_columns) staging_relation = relation.incorporate( path={"identifier": relation.identifier + "__dbt_tmp_migration_staging"} ) ctas = dedent( f"""\ select - {col_csv}, + {col_csv} , dbt_snapshot_at as dbt_updated_at, dbt_valid_from, if(dbt_valid_to > cast('9000-01-01' as timestamp), null, dbt_valid_to) as dbt_valid_to, dbt_scd_id from {relation} where dbt_change_type != 'delete' - ; - """ + ;""" ) staging_sql = self.execute_macro( "create_table_as", kwargs=dict(temporary=True, relation=staging_relation, compiled_code=ctas) diff --git a/dbt-athena/tests/functional/adapter/test_unique_tmp_table_suffix.py b/dbt-athena/tests/functional/adapter/test_unique_tmp_table_suffix.py index 563e5dcb..0f188ce1 100644 --- a/dbt-athena/tests/functional/adapter/test_unique_tmp_table_suffix.py +++ b/dbt-athena/tests/functional/adapter/test_unique_tmp_table_suffix.py @@ -33,7 +33,7 @@ def test__unique_tmp_table_suffix(self, project, capsys): model_run_result_row_count_query = f"select count(*) as records from {project.test_schema}.{relation_name}" expected_unique_table_name_re = ( r"unique_tmp_table_suffix__dbt_tmp_" - r"[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}" + r"[0-9a-fA-F]{8}_[0-9a-fA-F]{4}_[0-9a-fA-F]{4}_[0-9a-fA-F]{4}_[0-9a-fA-F]{12}" ) first_model_run = run_dbt( From 8f6b1f0813500677bdc81e0b524df0b4c7db1496 Mon Sep 17 00:00:00 2001 From: David Lequin Date: Mon, 2 Dec 2024 17:35:37 +0000 Subject: [PATCH 3/4] Stop adding aliases to render_limited output (#749) --- dbt-athena/src/dbt/adapters/athena/relation.py | 1 + dbt-athena/tests/functional/adapter/test_empty.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dbt-athena/src/dbt/adapters/athena/relation.py b/dbt-athena/src/dbt/adapters/athena/relation.py index 7862c719..0de8d784 100644 --- a/dbt-athena/src/dbt/adapters/athena/relation.py +++ b/dbt-athena/src/dbt/adapters/athena/relation.py @@ -39,6 +39,7 @@ class AthenaRelation(BaseRelation): include_policy: Policy = field(default_factory=lambda: AthenaIncludePolicy()) s3_path_table_part: Optional[str] = None detailed_table_type: Optional[str] = None # table_type option from the table Parameters in Glue Catalog + require_alias: bool = False def render_hive(self) -> str: """ diff --git a/dbt-athena/tests/functional/adapter/test_empty.py b/dbt-athena/tests/functional/adapter/test_empty.py index b9523881..c79fdfc2 100644 --- a/dbt-athena/tests/functional/adapter/test_empty.py +++ b/dbt-athena/tests/functional/adapter/test_empty.py @@ -1,5 +1,9 @@ -from dbt.tests.adapter.empty.test_empty import BaseTestEmpty +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef class TestAthenaEmpty(BaseTestEmpty): pass + + +class TestAthenaEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): + pass From b52d304c63a00cc015a5354e88823713536d9444 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 2 Dec 2024 09:55:21 -0800 Subject: [PATCH 4/4] bump to 1.9.0rc1 (#762) --- dbt-athena-community/pyproject.toml | 4 ++-- dbt-athena/src/dbt/adapters/athena/__version__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt-athena-community/pyproject.toml b/dbt-athena-community/pyproject.toml index b356879b..c0a7e9b8 100644 --- a/dbt-athena-community/pyproject.toml +++ b/dbt-athena-community/pyproject.toml @@ -22,8 +22,8 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] # these versions should always match and they both should match the local version of dbt-athena -dependencies = ["dbt-athena==1.9.0b1"] -version = "1.9.0b1" +dependencies = ["dbt-athena==1.9.0rc1"] +version = "1.9.0rc1" [project.urls] Homepage = "https://github.com/dbt-labs/dbt-athena/dbt-athena" Documentation = "https://docs.getdbt.com" diff --git a/dbt-athena/src/dbt/adapters/athena/__version__.py b/dbt-athena/src/dbt/adapters/athena/__version__.py index a4077fff..c70591d8 100644 --- a/dbt-athena/src/dbt/adapters/athena/__version__.py +++ b/dbt-athena/src/dbt/adapters/athena/__version__.py @@ -1 +1 @@ -version = "1.9.0b1" +version = "1.9.0rc1"