From 7a6b2914039208ae52617163e024a0dc45ef8a5a Mon Sep 17 00:00:00 2001 From: Sindre Grindheim Date: Fri, 11 Feb 2022 13:52:29 +0100 Subject: [PATCH 1/6] Not dropping table for incremental full refresh with delta --- .../spark/macros/materializations/incremental/incremental.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index d0b6e89ba..d426c2daf 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -29,7 +29,9 @@ {% if existing_relation is none %} {% set build_sql = create_table_as(False, target_relation, sql) %} {% elif existing_relation.is_view or full_refresh_mode %} - {% do adapter.drop_relation(existing_relation) %} + {% if existing_relation.is_view or file_format != 'delta' %} + {% do adapter.drop_relation(existing_relation) %} + {% endif %} {% set build_sql = create_table_as(False, target_relation, sql) %} {% else %} {% do run_query(create_table_as(True, tmp_relation, sql)) %} From ca5f39bf03376983e9ecdea85ca19f0b40e9d33b Mon Sep 17 00:00:00 2001 From: grindheim Date: Fri, 11 Feb 2022 14:04:12 +0100 Subject: [PATCH 2/6] Updated changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d245797..97d1dbcd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## dbt-spark 1.0.1 (TBD) + +### Fixes +- Incremental materialization updated to not drop table first if full refresh for delta lake format, as it already runs _create or replace table_ ([#286](https://github.com/dbt-labs/dbt-spark/issues/286), [#287](https://github.com/dbt-labs/dbt-spark/pull/287/)) + +### Contributors +- [@grindheim](https://github.com/grindheim) ([#287](https://github.com/dbt-labs/dbt-spark/pull/287/)) + + ## dbt-spark 1.0.0 (Release TBD) ### Fixes From 42f24f65f1b87fd57578473870843dcf34e2d549 Mon Sep 17 00:00:00 2001 From: Sindre Grindheim Date: Tue, 28 Jun 2022 12:32:19 +0200 Subject: [PATCH 3/6] Simplified conditional logic according to suggestion --- .../macros/materializations/incremental/incremental.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 6834b73b2..64a3c2b59 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -26,12 +26,12 @@ {{ run_hooks(pre_hooks) }} + {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %} + {% if existing_relation is none %} {% set build_sql = create_table_as(False, target_relation, sql) %} - {% elif existing_relation.is_view or full_refresh_mode %} - {% if existing_relation.is_view or file_format != 'delta' %} - {% do adapter.drop_relation(existing_relation) %} - {% endif %} + {% elif existing_relation.is_view or (full_refresh_mode and not is_delta) %} + {% do adapter.drop_relation(existing_relation) %} {% set build_sql = create_table_as(False, target_relation, sql) %} {% else %} {% do run_query(create_table_as(True, tmp_relation, sql)) %} From d82827ac15fefaf12f3a4a8c17a1e66c49c0fa1e Mon Sep 17 00:00:00 2001 From: Sindre Grindheim Date: Tue, 28 Jun 2022 12:35:41 +0200 Subject: [PATCH 4/6] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e64758fca..5da4db7e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## dbt-spark 1.2.0 (tbd) +## dbt-spark 1.2.0b1 (June 24, 2022) ### Fixes - `adapter.get_columns_in_relation` (method) and `get_columns_in_relation` (macro) now return identical responses. The previous behavior of `get_columns_in_relation` (macro) is now represented by a new macro, `get_columns_in_relation_raw` ([#354](https://github.com/dbt-labs/dbt-spark/issues/354), [#355](https://github.com/dbt-labs/dbt-spark/pull/355)) From b8b879f3c54002c080b7a3e058ad3f3a61597e12 Mon Sep 17 00:00:00 2001 From: Sindre Grindheim Date: Mon, 4 Jul 2022 09:13:43 +0200 Subject: [PATCH 5/6] Only drop table if not delta table Co-authored-by: Jeremy Cohen --- .../macros/materializations/incremental/incremental.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 64a3c2b59..99cd31db1 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -30,8 +30,10 @@ {% if existing_relation is none %} {% set build_sql = create_table_as(False, target_relation, sql) %} - {% elif existing_relation.is_view or (full_refresh_mode and not is_delta) %} - {% do adapter.drop_relation(existing_relation) %} + {% elif existing_relation.is_view or full_refresh_mode %} + {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#} + {% do adapter.drop_relation(existing_relation) %} + {% endif %} {% set build_sql = create_table_as(False, target_relation, sql) %} {% else %} {% do run_query(create_table_as(True, tmp_relation, sql)) %} From df48a0d2bc63e6a6e1fabc06cb8318610025ebc4 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 4 Jul 2022 16:12:04 +0200 Subject: [PATCH 6/6] Update changelog, trigger CircleCI tests --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5da4db7e3..5e49e77d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,18 @@ +## dbt-spark 1.2.0rc1 (Release TBD) + +- Incremental materialization updated to not drop table first if full refresh for delta lake format, as it already runs _create or replace table_ ([#286](https://github.com/dbt-labs/dbt-spark/issues/286), [#287](https://github.com/dbt-labs/dbt-spark/pull/287/)) + +### Contributors +- [@grindheim](https://github.com/grindheim) ([#287](https://github.com/dbt-labs/dbt-spark/pull/287/)) + ## dbt-spark 1.2.0b1 (June 24, 2022) ### Fixes - `adapter.get_columns_in_relation` (method) and `get_columns_in_relation` (macro) now return identical responses. The previous behavior of `get_columns_in_relation` (macro) is now represented by a new macro, `get_columns_in_relation_raw` ([#354](https://github.com/dbt-labs/dbt-spark/issues/354), [#355](https://github.com/dbt-labs/dbt-spark/pull/355)) -- Incremental materialization updated to not drop table first if full refresh for delta lake format, as it already runs _create or replace table_ ([#286](https://github.com/dbt-labs/dbt-spark/issues/286), [#287](https://github.com/dbt-labs/dbt-spark/pull/287/)) ### Under the hood - Add `DBT_INVOCATION_ENV` environment variable to ODBC user agent string ([#366](https://github.com/dbt-labs/dbt-spark/pull/366)) -### Contributors -- [@grindheim](https://github.com/grindheim) ([#287](https://github.com/dbt-labs/dbt-spark/pull/287/)) - ## dbt-spark 1.1.0 (April 28, 2022) ### Features