From 7f795796abe4e15b8627d6cf086accce772b5685 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Tue, 13 Sep 2022 09:41:00 -0600 Subject: [PATCH 1/6] Temporary dev-requirements --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 129dbbe64..9a859cc05 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@dbeatty/lift-shift-array-macros#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@dbeatty/lift-shift-array-macros#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 bumpversion From d3374a6e80eb2339f5608b3090f91e1931701b48 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Tue, 13 Sep 2022 09:49:18 -0600 Subject: [PATCH 2/6] Changelog entry --- .changes/unreleased/Features-20220913-084836.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20220913-084836.yaml diff --git a/.changes/unreleased/Features-20220913-084836.yaml b/.changes/unreleased/Features-20220913-084836.yaml new file mode 100644 index 000000000..0a4bd275d --- /dev/null +++ b/.changes/unreleased/Features-20220913-084836.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Array macros +time: 2022-09-13T08:48:36.255365-06:00 +custom: + Author: graciegoheen dbeatty10 + Issue: "307" + PR: "308" From 9af905025e77e230f257a10fa6c69e68843dcbe7 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Tue, 13 Sep 2022 11:13:28 -0600 Subject: [PATCH 3/6] Implementations and tests for array macros --- .../bigquery/macros/utils/array_append.sql | 3 +++ .../bigquery/macros/utils/array_concat.sql | 3 +++ .../bigquery/macros/utils/array_construct.sql | 3 +++ .../macros/utils/cast_array_to_string.sql | 3 +++ tests/functional/adapter/utils/test_utils.py | 17 ++++++++++++++++- 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 dbt/include/bigquery/macros/utils/array_append.sql create mode 100644 dbt/include/bigquery/macros/utils/array_concat.sql create mode 100644 dbt/include/bigquery/macros/utils/array_construct.sql create mode 100644 dbt/include/bigquery/macros/utils/cast_array_to_string.sql diff --git a/dbt/include/bigquery/macros/utils/array_append.sql b/dbt/include/bigquery/macros/utils/array_append.sql new file mode 100644 index 000000000..78bd5cc43 --- /dev/null +++ b/dbt/include/bigquery/macros/utils/array_append.sql @@ -0,0 +1,3 @@ +{% macro bigquery__array_append(array, new_element) -%} + {{ array_concat(array, array_construct([new_element])) }} +{%- endmacro %} diff --git a/dbt/include/bigquery/macros/utils/array_concat.sql b/dbt/include/bigquery/macros/utils/array_concat.sql new file mode 100644 index 000000000..eff8f524a --- /dev/null +++ b/dbt/include/bigquery/macros/utils/array_concat.sql @@ -0,0 +1,3 @@ +{% macro bigquery__array_concat(array_1, array_2) -%} + array_concat({{ array_1 }}, {{ array_2 }}) +{%- endmacro %} diff --git a/dbt/include/bigquery/macros/utils/array_construct.sql b/dbt/include/bigquery/macros/utils/array_construct.sql new file mode 100644 index 000000000..7764e4dc8 --- /dev/null +++ b/dbt/include/bigquery/macros/utils/array_construct.sql @@ -0,0 +1,3 @@ +{% macro bigquery__array_construct(inputs, data_type) -%} + [ {{ inputs|join(' , ') }} ] +{%- endmacro %} diff --git a/dbt/include/bigquery/macros/utils/cast_array_to_string.sql b/dbt/include/bigquery/macros/utils/cast_array_to_string.sql new file mode 100644 index 000000000..af11517bf --- /dev/null +++ b/dbt/include/bigquery/macros/utils/cast_array_to_string.sql @@ -0,0 +1,3 @@ +{% macro bigquery__cast_array_to_string(array) %} + '['||(select string_agg(cast(element as string), ',') from unnest({{ array }}) element)||']' +{% endmacro %} diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 2a37ac6b7..b4567116c 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -1,5 +1,8 @@ import pytest -from dbt.tests.adapter.utils.base_utils import BaseUtils + +from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend +from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat +from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct from dbt.tests.adapter.utils.test_any_value import BaseAnyValue from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText @@ -27,6 +30,18 @@ class TestAnyValue(BaseAnyValue): pass +class TestArrayAppend(BaseArrayAppend): + pass + + +class TestArrayConcat(BaseArrayConcat): + pass + + +class TestArrayConstruct(BaseArrayConstruct): + pass + + class TestBoolOr(BaseBoolOr): pass From c48e032d8492061a18c22de26b50ff12d84b80d8 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Wed, 21 Sep 2022 19:20:56 -0600 Subject: [PATCH 4/6] Remove `cast_array_to_string` macro --- dbt/include/bigquery/macros/utils/cast_array_to_string.sql | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 dbt/include/bigquery/macros/utils/cast_array_to_string.sql diff --git a/dbt/include/bigquery/macros/utils/cast_array_to_string.sql b/dbt/include/bigquery/macros/utils/cast_array_to_string.sql deleted file mode 100644 index af11517bf..000000000 --- a/dbt/include/bigquery/macros/utils/cast_array_to_string.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro bigquery__cast_array_to_string(array) %} - '['||(select string_agg(cast(element as string), ',') from unnest({{ array }}) element)||']' -{% endmacro %} From 8242425920b2ce557adf14e3eef0709365200609 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 23 Sep 2022 18:10:28 -0600 Subject: [PATCH 5/6] ARRAY in BigQuery does not support set operation comparisons using EXCEPT --- .../bigquery/macros/utils/array_construct.sql | 4 ++ .../adapter/utils/fixture_array_append.py | 13 +++++ .../adapter/utils/fixture_array_concat.py | 15 ++++++ .../adapter/utils/fixture_array_construct.py | 20 +++++++ tests/functional/adapter/utils/test_utils.py | 52 +++++++++++++++++-- 5 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 tests/functional/adapter/utils/fixture_array_append.py create mode 100644 tests/functional/adapter/utils/fixture_array_concat.py create mode 100644 tests/functional/adapter/utils/fixture_array_construct.py diff --git a/dbt/include/bigquery/macros/utils/array_construct.sql b/dbt/include/bigquery/macros/utils/array_construct.sql index 7764e4dc8..270b1f785 100644 --- a/dbt/include/bigquery/macros/utils/array_construct.sql +++ b/dbt/include/bigquery/macros/utils/array_construct.sql @@ -1,3 +1,7 @@ {% macro bigquery__array_construct(inputs, data_type) -%} + {% if inputs|length > 0 %} [ {{ inputs|join(' , ') }} ] + {% else %} + ARRAY<{{data_type}}>[] + {% endif %} {%- endmacro %} diff --git a/tests/functional/adapter/utils/fixture_array_append.py b/tests/functional/adapter/utils/fixture_array_append.py new file mode 100644 index 000000000..0558d66e1 --- /dev/null +++ b/tests/functional/adapter/utils/fixture_array_append.py @@ -0,0 +1,13 @@ +# array_append + +# EXCEPT can't be used with ARRAYs in BigQuery, so convert to a string +models__array_append_expected_sql = """ +select 1 as id, {{ array_to_string(array_construct([1,2,3,4])) }} as array_col union all +select 2 as id, {{ array_to_string(array_construct([4])) }} as array_col +""" + + +models__array_append_actual_sql = """ +select 1 as id, {{ array_to_string(array_append(array_construct([1,2,3]), 4)) }} as array_col union all +select 2 as id, {{ array_to_string(array_append(array_construct([]), 4)) }} as array_col +""" diff --git a/tests/functional/adapter/utils/fixture_array_concat.py b/tests/functional/adapter/utils/fixture_array_concat.py new file mode 100644 index 000000000..51af8bf12 --- /dev/null +++ b/tests/functional/adapter/utils/fixture_array_concat.py @@ -0,0 +1,15 @@ +# array_concat + +# EXCEPT can't be used with ARRAYs in BigQuery, so convert to a string +models__array_concat_expected_sql = """ +select 1 as id, {{ array_to_string(array_construct([1,2,3,4,5,6])) }} as array_col union all +select 2 as id, {{ array_to_string(array_construct([2])) }} as array_col union all +select 3 as id, {{ array_to_string(array_construct([3])) }} as array_col +""" + + +models__array_concat_actual_sql = """ +select 1 as id, {{ array_to_string(array_concat(array_construct([1,2,3]), array_construct([4,5,6]))) }} as array_col union all +select 2 as id, {{ array_to_string(array_concat(array_construct([]), array_construct([2]))) }} as array_col union all +select 3 as id, {{ array_to_string(array_concat(array_construct([3]), array_construct([]))) }} as array_col +""" diff --git a/tests/functional/adapter/utils/fixture_array_construct.py b/tests/functional/adapter/utils/fixture_array_construct.py new file mode 100644 index 000000000..13d0bb2f3 --- /dev/null +++ b/tests/functional/adapter/utils/fixture_array_construct.py @@ -0,0 +1,20 @@ +# array_construct + +# EXCEPT can't be used with ARRAYs in BigQuery, so convert to a string +models__array_construct_expected_sql = """ +select 1 as id, {{ array_to_string(array_construct([1,2,3])) }} as array_col union all +select 2 as id, {{ array_to_string(array_construct([])) }} as array_col +""" + + +models__array_construct_actual_sql = """ +select 1 as id, {{ array_to_string(array_construct([1,2,3])) }} as array_col union all +select 2 as id, {{ array_to_string(array_construct([])) }} as array_col +""" + + +macros__array_to_string_sql = """ +{% macro array_to_string(array) %} + (select string_agg(cast(element as string), ',') from unnest({{ array }}) element) +{% endmacro %} +""" diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index b4567116c..295b49535 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -24,6 +24,19 @@ from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral +from tests.functional.adapter.utils.fixture_array_append import ( + models__array_append_actual_sql, + models__array_append_expected_sql, +) +from tests.functional.adapter.utils.fixture_array_concat import ( + models__array_concat_actual_sql, + models__array_concat_expected_sql, +) +from tests.functional.adapter.utils.fixture_array_construct import ( + models__array_construct_actual_sql, + models__array_construct_expected_sql, + macros__array_to_string_sql, +) class TestAnyValue(BaseAnyValue): @@ -31,15 +44,48 @@ class TestAnyValue(BaseAnyValue): class TestArrayAppend(BaseArrayAppend): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "actual.sql": models__array_append_actual_sql, + "expected.sql": models__array_append_expected_sql, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "array_to_string.sql": macros__array_to_string_sql, + } class TestArrayConcat(BaseArrayConcat): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "actual.sql": models__array_concat_actual_sql, + "expected.sql": models__array_concat_expected_sql, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "array_to_string.sql": macros__array_to_string_sql, + } class TestArrayConstruct(BaseArrayConstruct): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "actual.sql": models__array_construct_actual_sql, + "expected.sql": models__array_construct_expected_sql, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "array_to_string.sql": macros__array_to_string_sql, + } class TestBoolOr(BaseBoolOr): From 739d1ea02e09d057c532ab3b66f13ed8485611cd Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Mon, 26 Sep 2022 13:44:47 -0600 Subject: [PATCH 6/6] Restore original dev-requirements.txt --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f3a623292..676703d3e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@dbeatty/lift-shift-array-macros#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@dbeatty/lift-shift-array-macros#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 bumpversion