From 5058049096db9c03195481f292ec8e64452e5690 Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Mon, 2 Dec 2019 10:45:26 -0700 Subject: [PATCH 01/14] Add support for sql as a header to create or replace --- plugins/bigquery/dbt/include/bigquery/macros/adapters.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index 944127024f5..2f4c0142703 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -26,6 +26,10 @@ {%- endmacro -%} +{% macro sql_header(config) -%} + {{ config.set('sql_header', caller()) }} +{%- endmacro %} + {% macro bigquery_table_options(persist_docs, temporary, kms_key_name) %} {% set opts = {} %} @@ -53,6 +57,9 @@ {%- set raw_cluster_by = config.get('cluster_by', none) -%} {%- set raw_persist_docs = config.get('persist_docs', {}) -%} {%- set raw_kms_key_name = config.get('kms_key_name', none) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create or replace table {{ relation }} {{ partition_by(raw_partition_by) }} From 5eafbb794a3b1b5c268989c0b949c0ced4da7162 Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Tue, 3 Dec 2019 16:01:52 -0700 Subject: [PATCH 02/14] Make sql_header universal accross adapters --- .../include/global_project/macros/adapters/common.sql | 4 ++++ core/dbt/source_config.py | 2 +- .../bigquery/dbt/include/bigquery/macros/adapters.sql | 7 +++---- .../postgres/dbt/include/postgres/macros/adapters.sql | 3 +++ .../redshift/dbt/include/redshift/macros/adapters.sql | 6 ++++++ .../022_bigquery_test/models/sql_header_model.sql | 11 +++++++++++ 6 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 test/integration/022_bigquery_test/models/sql_header_model.sql diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql index 6bf15509175..bd69180bb0f 100644 --- a/core/dbt/include/global_project/macros/adapters/common.sql +++ b/core/dbt/include/global_project/macros/adapters/common.sql @@ -269,3 +269,7 @@ {% do return(tmp_relation) %} {% endmacro %} + +{% macro set_sql_header(config) -%} + {{ config.set('sql_header', caller()) }} +{%- endmacro %} \ No newline at end of file diff --git a/core/dbt/source_config.py b/core/dbt/source_config.py index eacbbabcad9..dd1efa00bdb 100644 --- a/core/dbt/source_config.py +++ b/core/dbt/source_config.py @@ -16,7 +16,7 @@ class SourceConfig: 'unique_key', 'database', 'severity', - + 'sql_header', 'incremental_strategy', # snapshots diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index 2f4c0142703..1042665b3ef 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -26,10 +26,6 @@ {%- endmacro -%} -{% macro sql_header(config) -%} - {{ config.set('sql_header', caller()) }} -{%- endmacro %} - {% macro bigquery_table_options(persist_docs, temporary, kms_key_name) %} {% set opts = {} %} @@ -73,6 +69,9 @@ {% macro bigquery__create_view_as(relation, sql) -%} {%- set raw_persist_docs = config.get('persist_docs', {}) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create or replace view {{ relation }} {{ bigquery_table_options(persist_docs=raw_persist_docs, temporary=false) }} diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 82484b34978..3fb3b886452 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -1,5 +1,8 @@ {% macro postgres__create_table_as(temporary, relation, sql) -%} {%- set unlogged = config.get('unlogged', default=false) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create {% if temporary -%} temporary diff --git a/plugins/redshift/dbt/include/redshift/macros/adapters.sql b/plugins/redshift/dbt/include/redshift/macros/adapters.sql index fa75765c0a3..5e1f203af2b 100644 --- a/plugins/redshift/dbt/include/redshift/macros/adapters.sql +++ b/plugins/redshift/dbt/include/redshift/macros/adapters.sql @@ -37,6 +37,9 @@ {%- set _sort = config.get( 'sort', validator=validation.any[list, basestring]) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create {% if temporary -%}temporary{%- endif %} table {{ relation.include(database=(not temporary), schema=(not temporary)) }} @@ -51,6 +54,9 @@ {% macro redshift__create_view_as(relation, sql) -%} {% set bind_qualifier = '' if config.get('bind', default=True) else 'with no schema binding' %} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create view {{ relation }} as ( {{ sql }} diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql new file mode 100644 index 00000000000..c5744a8c8c9 --- /dev/null +++ b/test/integration/022_bigquery_test/models/sql_header_model.sql @@ -0,0 +1,11 @@ +{# This will fail if it is not extracted correctly #} +{% call set_sql_header(config) %} + create or replace table {{ ref('table_model') }} + OPTIONS( + persist_docs={ "relation": true, "columns": true, "schema": true }) + as ( + select * from {{ ref('view_model') }} + ) +{% endcall %} + +select * from {{ ref('view_model') }} \ No newline at end of file From 9e9f0307df95eb2edfc7c6b3bf2716b23e4e144e Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Tue, 3 Dec 2019 16:04:19 -0700 Subject: [PATCH 03/14] newline and better test --- .../include/global_project/macros/adapters/common.sql | 2 +- .../022_bigquery_test/models/sql_header_model.sql | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql index bd69180bb0f..dd01f56a670 100644 --- a/core/dbt/include/global_project/macros/adapters/common.sql +++ b/core/dbt/include/global_project/macros/adapters/common.sql @@ -272,4 +272,4 @@ {% macro set_sql_header(config) -%} {{ config.set('sql_header', caller()) }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql index c5744a8c8c9..81880792444 100644 --- a/test/integration/022_bigquery_test/models/sql_header_model.sql +++ b/test/integration/022_bigquery_test/models/sql_header_model.sql @@ -1,11 +1,8 @@ {# This will fail if it is not extracted correctly #} {% call set_sql_header(config) %} - create or replace table {{ ref('table_model') }} - OPTIONS( - persist_docs={ "relation": true, "columns": true, "schema": true }) - as ( - select * from {{ ref('view_model') }} - ) + create or replace table {{ ref('table_model') }} as ( + select * from {{ ref('table_model') }} + ) {% endcall %} select * from {{ ref('view_model') }} \ No newline at end of file From 7a51ef664a2e13593a7f7a97666e7ba7a213420b Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Tue, 3 Dec 2019 16:04:58 -0700 Subject: [PATCH 04/14] Another newline --- test/integration/022_bigquery_test/models/sql_header_model.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql index 81880792444..dec84789e96 100644 --- a/test/integration/022_bigquery_test/models/sql_header_model.sql +++ b/test/integration/022_bigquery_test/models/sql_header_model.sql @@ -5,4 +5,4 @@ ) {% endcall %} -select * from {{ ref('view_model') }} \ No newline at end of file +select * from {{ ref('view_model') }} From fc94a5e19d8b4e22034797775619fe56410fb45a Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Wed, 4 Dec 2019 09:21:03 -0700 Subject: [PATCH 05/14] Add snowflake support and test UDF --- .../dbt/include/snowflake/macros/adapters.sql | 6 ++++++ .../022_bigquery_test/models/sql_header_model.sql | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql index 5cf8b7b63cd..afd39f085c8 100644 --- a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql +++ b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql @@ -11,6 +11,9 @@ {% else %} {%- set cluster_by_string = none -%} {%- endif -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create or replace {% if temporary -%} temporary @@ -38,6 +41,9 @@ {% macro snowflake__create_view_as(relation, sql) -%} {%- set secure = config.get('secure', default=false) -%} {%- set copy_grants = config.get('copy_grants', default=false) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} create or replace {% if secure -%} secure {%- endif %} view {{ relation }} {% if copy_grants -%} copy grants {%- endif %} as ( diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql index dec84789e96..687669d3a7d 100644 --- a/test/integration/022_bigquery_test/models/sql_header_model.sql +++ b/test/integration/022_bigquery_test/models/sql_header_model.sql @@ -1,8 +1,12 @@ {# This will fail if it is not extracted correctly #} {% call set_sql_header(config) %} - create or replace table {{ ref('table_model') }} as ( - select * from {{ ref('table_model') }} - ) + CREATE TEMPORARY FUNCTION a_to_b(str STRING) + RETURNS STRING AS ( + CASE + WHEN LOWER(str) = 'a' THEN 'b' + ELSE str + END + ) {% endcall %} -select * from {{ ref('view_model') }} +select a_to_b(dupe) from {{ ref('view_model') }} From 0f05404f72c8ea9aeaea909f1aa431ad55300efe Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Mon, 9 Dec 2019 14:45:02 -0700 Subject: [PATCH 06/14] Add sql_header common default --- .../dbt/include/global_project/macros/adapters/common.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql index dd01f56a670..db30c2ba286 100644 --- a/core/dbt/include/global_project/macros/adapters/common.sql +++ b/core/dbt/include/global_project/macros/adapters/common.sql @@ -69,6 +69,10 @@ {%- endmacro %} {% macro default__create_table_as(temporary, relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=(not temporary), schema=(not temporary)) }} as ( @@ -81,6 +85,10 @@ {%- endmacro %} {% macro default__create_view_as(relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create view {{ relation }} as ( {{ sql }} ); From d9ee6ea91773782eefcc2ddcbe48064a80af6b1f Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Mon, 16 Dec 2019 11:02:44 -0700 Subject: [PATCH 07/14] Add semicolon after UDF --- test/integration/022_bigquery_test/models/sql_header_model.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql index 687669d3a7d..e48e43d14d4 100644 --- a/test/integration/022_bigquery_test/models/sql_header_model.sql +++ b/test/integration/022_bigquery_test/models/sql_header_model.sql @@ -6,7 +6,7 @@ WHEN LOWER(str) = 'a' THEN 'b' ELSE str END - ) + ); {% endcall %} select a_to_b(dupe) from {{ ref('view_model') }} From c4baf72fd49adae4d86866b3b28f191b71ec2668 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 18 Dec 2019 09:55:15 -0500 Subject: [PATCH 08/14] Update CHANGELOG.md --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c990ec1573d..38a8a3c1cb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## dbt 0.15.1 (To be released) + +This is a bugfix release. + +### Features +- Lazily load database connections ([#1584](https://github.com/fishtown-analytics/dbt/issues/1584), [#1992](https://github.com/fishtown-analytics/dbt/pull/1992)) +- Support raising warnings in user-space ([#1970](https://github.com/fishtown-analytics/dbt/issues/1970), [#1977](https://github.com/fishtown-analytics/dbt/pull/1977)) + +### Fixes +- Fix for catalog generation error when datasets are not missing on BigQuery ([#1984](https://github.com/fishtown-analytics/dbt/issues/1984), [#2005](https://github.com/fishtown-analytics/dbt/pull/2005)) +- Fix for invalid SQL generated when "check" strategy is used in Snapshots with changing schemas ([#1797](https://github.com/fishtown-analytics/dbt/issues/1797), [#2001](https://github.com/fishtown-analytics/dbt/pull/2001)( +- Fix for gaps in valid_from and valid_to timestamps when "check" strategy is used in Snapshots on some databases ([#1736](https://github.com/fishtown-analytics/dbt/issues/1736), [#1994](https://github.com/fishtown-analytics/dbt/pull/1994)) +- Fix incorrect thread names in dbt server logs ([#1905](https://github.com/fishtown-analytics/dbt/issues/1905), [#2002](https://github.com/fishtown-analytics/dbt/pull/2002)) +- Fix for ignored catalog data when user schemas begin with `pg*` on Postgres and Redshift ([#1960](https://github.com/fishtown-analytics/dbt/issues/1960), [#2003](https://github.com/fishtown-analytics/dbt/pull/2003)) +- Fix for poorly defined materialization resolution logic ([#1962](https://github.com/fishtown-analytics/dbt/issues/1962), [#1976](https://github.com/fishtown-analytics/dbt/pull/1976)) +- Fix missing `drop_schema` method in adapter namespace ([#1980](https://github.com/fishtown-analytics/dbt/issues/1980), [#1983](https://github.com/fishtown-analytics/dbt/pull/1983)) + +### Under the hood +- Fail more gracefully at install time when setuptools is downlevel ([#1975](https://github.com/fishtown-analytics/dbt/issues/1975), [#1978](https://github.com/fishtown-analytics/dbt/pull/1978)) +- Make the `DBT_TEST_ALT` integration test warehouse configurable on Snowflake ([#1939](https://github.com/fishtown-analytics/dbt/issues/1939), [#1979](https://github.com/fishtown-analytics/dbt/pull/1979)) + + ## dbt 0.15.0 (November 25, 2019) ### Breaking changes From 0b05bf0fa64cc94bae6c9f710fbef5465de92322 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 18 Dec 2019 11:59:59 -0500 Subject: [PATCH 09/14] Update CHANGELOG.md Co-Authored-By: Jacob Beck --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38a8a3c1cb9..e8c41af692e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ This is a bugfix release. - Support raising warnings in user-space ([#1970](https://github.com/fishtown-analytics/dbt/issues/1970), [#1977](https://github.com/fishtown-analytics/dbt/pull/1977)) ### Fixes -- Fix for catalog generation error when datasets are not missing on BigQuery ([#1984](https://github.com/fishtown-analytics/dbt/issues/1984), [#2005](https://github.com/fishtown-analytics/dbt/pull/2005)) +- Fix for catalog generation error when datasets are missing on BigQuery ([#1984](https://github.com/fishtown-analytics/dbt/issues/1984), [#2005](https://github.com/fishtown-analytics/dbt/pull/2005)) - Fix for invalid SQL generated when "check" strategy is used in Snapshots with changing schemas ([#1797](https://github.com/fishtown-analytics/dbt/issues/1797), [#2001](https://github.com/fishtown-analytics/dbt/pull/2001)( - Fix for gaps in valid_from and valid_to timestamps when "check" strategy is used in Snapshots on some databases ([#1736](https://github.com/fishtown-analytics/dbt/issues/1736), [#1994](https://github.com/fishtown-analytics/dbt/pull/1994)) - Fix incorrect thread names in dbt server logs ([#1905](https://github.com/fishtown-analytics/dbt/issues/1905), [#2002](https://github.com/fishtown-analytics/dbt/pull/2002)) From 27212a16ddd074d39773bccbb189825d6ffc7317 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 18 Dec 2019 15:33:20 -0500 Subject: [PATCH 10/14] Update CHANGELOG.md --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8c41af692e..9052f21dbdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This is a bugfix release. ### Features - Lazily load database connections ([#1584](https://github.com/fishtown-analytics/dbt/issues/1584), [#1992](https://github.com/fishtown-analytics/dbt/pull/1992)) - Support raising warnings in user-space ([#1970](https://github.com/fishtown-analytics/dbt/issues/1970), [#1977](https://github.com/fishtown-analytics/dbt/pull/1977)) +- Suppport BigQuery label configuration for models ([#1942](https://github.com/fishtown-analytics/dbt/issues/1942), [#1964](https://github.com/fishtown-analytics/dbt/pull/1964)) +- Support retrying when BigQuery models fail with server errors ([#1579](https://github.com/fishtown-analytics/dbt/issues/1579), [#1963](https://github.com/fishtown-analytics/dbt/pull/1963)) ### Fixes - Fix for catalog generation error when datasets are missing on BigQuery ([#1984](https://github.com/fishtown-analytics/dbt/issues/1984), [#2005](https://github.com/fishtown-analytics/dbt/pull/2005)) @@ -14,10 +16,18 @@ This is a bugfix release. - Fix for ignored catalog data when user schemas begin with `pg*` on Postgres and Redshift ([#1960](https://github.com/fishtown-analytics/dbt/issues/1960), [#2003](https://github.com/fishtown-analytics/dbt/pull/2003)) - Fix for poorly defined materialization resolution logic ([#1962](https://github.com/fishtown-analytics/dbt/issues/1962), [#1976](https://github.com/fishtown-analytics/dbt/pull/1976)) - Fix missing `drop_schema` method in adapter namespace ([#1980](https://github.com/fishtown-analytics/dbt/issues/1980), [#1983](https://github.com/fishtown-analytics/dbt/pull/1983)) +- Fix incorrect `generated_at` value in the catalog ([#1988](https://github.com/fishtown-analytics/dbt/pull/1988)) ### Under the hood - Fail more gracefully at install time when setuptools is downlevel ([#1975](https://github.com/fishtown-analytics/dbt/issues/1975), [#1978](https://github.com/fishtown-analytics/dbt/pull/1978)) - Make the `DBT_TEST_ALT` integration test warehouse configurable on Snowflake ([#1939](https://github.com/fishtown-analytics/dbt/issues/1939), [#1979](https://github.com/fishtown-analytics/dbt/pull/1979)) +- Pin upper bound on `google-cloud-bigquery` dependency to `1.24.0`. ([#2007](https://github.com/fishtown-analytics/dbt/pull/2007)) +- Remove duplicate `get_context_modules` method ([#1996](https://github.com/fishtown-analytics/dbt/pull/1996)) +- Add type annotations to base adapter code ([#1982](https://github.com/fishtown-analytics/dbt/pull/1982)) + +Contributors: + - [@Fokko](https://github.com/Fokko) ([#1996](https://github.com/fishtown-analytics/dbt/pull/1996), [#1988](https://github.com/fishtown-analytics/dbt/pull/1988), [#1982](https://github.com/fishtown-analytics/dbt/pull/1982)) + ## dbt 0.15.0 (November 25, 2019) From 6be4ee00aaf5e794005835d0144e4e6a3e87834e Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 18 Dec 2019 15:37:30 -0500 Subject: [PATCH 11/14] =?UTF-8?q?Bump=20version:=200.15.0=20=E2=86=92=200.?= =?UTF-8?q?15.1rc1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- core/dbt/version.py | 2 +- core/setup.py | 2 +- plugins/bigquery/setup.py | 2 +- plugins/postgres/setup.py | 2 +- plugins/redshift/setup.py | 2 +- plugins/snowflake/setup.py | 2 +- setup.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 482a5881ba7..106844f81eb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.15.0 +current_version = 0.15.1rc1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) diff --git a/core/dbt/version.py b/core/dbt/version.py index 3db55fe6f9c..7bf24e78ee2 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -56,5 +56,5 @@ def get_version_information(): .format(version_msg)) -__version__ = '0.15.0' +__version__ = '0.15.1rc1' installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index 399d6ec74f3..f0fdfeab51f 100644 --- a/core/setup.py +++ b/core/setup.py @@ -18,7 +18,7 @@ def read(fname): package_name = "dbt-core" -package_version = "0.15.0" +package_version = "0.15.1rc1" description = """dbt (data build tool) is a command line tool that helps \ analysts and engineers transform data in their warehouse more effectively""" diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index eb723820e8a..a54ef6b1a73 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-bigquery" -package_version = "0.15.0" +package_version = "0.15.1rc1" description = """The bigquery adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index 89d86788902..c0eaf5c6b3b 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -35,7 +35,7 @@ def _dbt_psycopg2_name(): package_name = "dbt-postgres" -package_version = "0.15.0" +package_version = "0.15.1rc1" description = """The postgres adpter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py index d241e9409b6..c1672811b49 100644 --- a/plugins/redshift/setup.py +++ b/plugins/redshift/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-redshift" -package_version = "0.15.0" +package_version = "0.15.1rc1" description = """The redshift adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py index a060ae13e28..dad91bf0648 100644 --- a/plugins/snowflake/setup.py +++ b/plugins/snowflake/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-snowflake" -package_version = "0.15.0" +package_version = "0.15.1rc1" description = """The snowflake adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/setup.py b/setup.py index 715551ce64d..5516628c517 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ package_name = "dbt" -package_version = "0.15.0" +package_version = "0.15.1rc1" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications.""" From 98c6eace0c46c59deff011ed1950cd990c10a150 Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Wed, 18 Dec 2019 14:26:37 -0700 Subject: [PATCH 12/14] materialize test model as table --- test/integration/022_bigquery_test/models/sql_header_model.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/022_bigquery_test/models/sql_header_model.sql b/test/integration/022_bigquery_test/models/sql_header_model.sql index 1e7983b0d2a..e49d82c4bc0 100644 --- a/test/integration/022_bigquery_test/models/sql_header_model.sql +++ b/test/integration/022_bigquery_test/models/sql_header_model.sql @@ -1,3 +1,5 @@ +{{ config(materialized="table") }} + {# This will fail if it is not extracted correctly #} {% call set_sql_header(config) %} CREATE TEMPORARY FUNCTION a_to_b(str STRING) From e6ab64ee454f93e158a168d6c14b7da9129cb2e2 Mon Sep 17 00:00:00 2001 From: Kurt Convey Date: Thu, 19 Dec 2019 09:39:50 -0700 Subject: [PATCH 13/14] Fix expected number of test models --- .../022_bigquery_test/test_simple_bigquery_view.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/022_bigquery_test/test_simple_bigquery_view.py b/test/integration/022_bigquery_test/test_simple_bigquery_view.py index b1b0dd9b58f..e25704ba1b4 100644 --- a/test/integration/022_bigquery_test/test_simple_bigquery_view.py +++ b/test/integration/022_bigquery_test/test_simple_bigquery_view.py @@ -53,7 +53,7 @@ def test__bigquery_simple_run(self): self.run_dbt(['seed', '--full-refresh']) results = self.run_dbt() # Bump expected number of results when adding new model - self.assertEqual(len(results), 7) + self.assertEqual(len(results), 8) self.assert_nondupes_pass() @@ -64,7 +64,7 @@ class TestUnderscoreBigQueryRun(TestBaseBigQueryRun): def test_bigquery_run_twice(self): self.run_dbt(['seed']) results = self.run_dbt() - self.assertEqual(len(results), 7) + self.assertEqual(len(results), 8) results = self.run_dbt() - self.assertEqual(len(results), 7) + self.assertEqual(len(results), 8) self.assert_nondupes_pass() From 86b917fd33bcbc27084f5d36236df27c85f4dae0 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 19 Dec 2019 10:22:10 -0700 Subject: [PATCH 14/14] fixes --- .../dbt/adapters/bigquery/connections.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py index 08ee91ee541..35146bee24f 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/connections.py +++ b/plugins/bigquery/dbt/adapters/bigquery/connections.py @@ -176,9 +176,12 @@ def get_timeout(cls, conn): return credentials.timeout_seconds @classmethod - def get_retries(cls, conn): + def get_retries(cls, conn) -> int: credentials = conn.credentials - return credentials.retries + if credentials.retries is not None: + return credentials.retries + else: + return 1 @classmethod def get_table_from_response(cls, resp): @@ -270,8 +273,11 @@ def create_table(self, database, schema, table_name, sql): job_params = {'destination': table_ref, 'write_disposition': 'WRITE_TRUNCATE'} + timeout = self.get_timeout(conn) + def fn(): - return self._query_and_results(client, sql, conn, job_params) + return self._query_and_results(client, sql, conn, job_params, + timeout=timeout) self._retry_and_handle(msg=sql, conn=conn, fn=fn) def create_date_partitioned_table(self, database, schema, table_name): @@ -317,12 +323,12 @@ def fn(): return client.create_dataset(dataset, exists_ok=True) self._retry_and_handle(msg='create dataset', conn=conn, fn=fn) - def _query_and_results(self, client, sql, conn, job_params): + def _query_and_results(self, client, sql, conn, job_params, timeout=None): """Query the client and wait for results.""" # Cannot reuse job_config if destination is set and ddl is used job_config = google.cloud.bigquery.QueryJobConfig(**job_params) query_job = client.query(sql, job_config=job_config) - iterator = query_job.result(timeout=self.get_timeout(conn)) + iterator = query_job.result(timeout=timeout) return query_job, iterator @@ -354,13 +360,13 @@ def count_error(self, error): return False # Don't log self.error_count += 1 if _is_retryable(error) and self.error_count <= self.retries: - logger.warning( - 'Retry attempt %s of %s after error: %s', + logger.debug( + 'Retry attempt {} of {} after error: {}', self.error_count, self.retries, repr(error)) return True else: - logger.warning( - 'Not Retrying after %s previous attempts. Error: %s', + logger.debug( + 'Not Retrying after {} previous attempts. Error: {}', self.error_count - 1, repr(error)) return False