From 9b71a47a67f3b162687a985c8d51b13645bbf87f Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Wed, 5 Apr 2023 21:31:45 -0400 Subject: [PATCH 1/8] CT-2221: Temporary change to dev-reqs --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index d0ee86a79..fc12cd0fd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@paw/ct-1922-model-level-constraints#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@paw/ct-1922-model-level-constraints#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor From d52d40e67a59998bed9c94d706403300438f3645 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 6 Apr 2023 10:10:09 -0400 Subject: [PATCH 2/8] CT-2221: Add BigQuery adapter support for model-level constraints --- dbt/adapters/bigquery/impl.py | 13 +++++++ dbt/include/bigquery/macros/adapters.sql | 2 +- .../macros/utils/get_columns_spec_ddl.sql | 31 ---------------- tests/functional/adapter/test_constraints.py | 35 +++++++++++++++++-- 4 files changed, 47 insertions(+), 34 deletions(-) delete mode 100644 dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 158cbc151..63ac97135 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -1,6 +1,8 @@ from dataclasses import dataclass import threading from typing import Dict, List, Optional, Any, Set, Union, Type + +from dbt.contracts.graph.nodes import ColumnLevelConstraint, ModelLevelConstraint, ConstraintType from dbt.dataclass_schema import dbtClassMixin, ValidationError import dbt.deprecations @@ -907,3 +909,14 @@ def python_submission_helpers(self) -> Dict[str, Type[PythonJobHelper]]: "cluster": ClusterDataprocHelper, "serverless": ServerlessDataProcHelper, } + + @classmethod + def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str: + if constraint.type != ConstraintType.not_null: + return "" # Only not null column constraints are supported by BigQuery + else: + return super().render_column_constraint(constraint) + + @classmethod + def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]: + return None # BigQuery doesn't currently support table constraints diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index dc1335db9..bb7fb8b1a 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -55,7 +55,7 @@ {%- set contract_config = config.get('contract') -%} {%- if contract_config.enforced -%} {{ get_assert_columns_equivalent(compiled_code) }} - {{ get_columns_spec_ddl() }} + {{ get_table_columns_and_constraints() }} {%- set compiled_code = get_select_subquery(compiled_code) %} {% endif %} {{ partition_by(partition_config) }} diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql deleted file mode 100644 index 091209ae3..000000000 --- a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql +++ /dev/null @@ -1,31 +0,0 @@ -{% macro bigquery__get_columns_spec_ddl() %} - {# loop through user_provided_columns to create DDL with data types and constraints #} - {%- set ns = namespace(at_least_one_check=False, at_least_one_pk=False) -%} - {%- set user_provided_columns = model['columns'] -%} - ( - {% for i in user_provided_columns %} - {%- set col = user_provided_columns[i] -%} - {%- set constraints = col['constraints'] -%} - {{ col['name'] }} {{ col['data_type'] }} - {%- for c in constraints -%} - {%- if c.type == "check" -%} - {%- set ns.at_least_one_check = True -%} - {%- elif c.type == "primary_key" -%} - {%- set ns.at_least_one_pk = True -%} - {%- else %} {{ adapter.render_raw_column_constraint(c) }} - {%- endif -%} - {%- endfor -%} - {{ "," if not loop.last }} - {% endfor -%} - ) - {%- if ns.at_least_one_check -%} - {{exceptions.warn("We noticed you have check constraints in your configs. These are not compatible with BigQuery and will be ignored.")}} - {%- endif -%} - {%- if ns.at_least_one_pk -%} - {{exceptions.warn("We noticed you have primary key constraints in your configs. These are not compatible with BigQuery and will be ignored.")}} - {%- endif -%} -{% endmacro %} - -{% macro bigquery__format_column(column) -%} - {{ return(column.column.lower() ~ " " ~ column.data_type) }} -{%- endmacro -%} diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 266b97b0d..650a8f586 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -7,7 +7,7 @@ BaseConstraintsRuntimeDdlEnforcement, BaseConstraintsRollback, BaseIncrementalConstraintsRuntimeDdlEnforcement, - BaseIncrementalConstraintsRollback, + BaseIncrementalConstraintsRollback, BaseModelConstraintsRuntimeEnforcement, ) from dbt.tests.adapter.constraints.fixtures import ( my_model_sql, @@ -167,4 +167,35 @@ def models(self): @pytest.fixture(scope="class") def expected_error_messages(self): - return ["Required field id cannot be null"] \ No newline at end of file + return ["Required field id cannot be null"] + + +class TestBigQueryModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": constraints_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create or replace table ( + id integer not null, + color string, + date_day string +) +OPTIONS() +as ( + select id, + color, + date_day from + ( + select 1 as id, + 'blue' as color, + '2019-01-01' as date_day + ) as model_subq +); +""" \ No newline at end of file From 761598d841746168e6f68b4614610234d71dbb64 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 6 Apr 2023 10:45:17 -0400 Subject: [PATCH 3/8] CT-2221: Add changelog entry --- .changes/unreleased/Features-20230406-104433.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230406-104433.yaml diff --git a/.changes/unreleased/Features-20230406-104433.yaml b/.changes/unreleased/Features-20230406-104433.yaml new file mode 100644 index 000000000..4ec1d8120 --- /dev/null +++ b/.changes/unreleased/Features-20230406-104433.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support for model-level constraints +time: 2023-04-06T10:44:33.045896-04:00 +custom: + Author: peterallenwebb + Issue: "569" From 25e9670d8cf6f7ed2c8431115fa8c26e946e629f Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 6 Apr 2023 17:40:06 -0400 Subject: [PATCH 4/8] CT-2221: Fixed unit tests. Added support for NOT ENFORCED primary key syntax. --- dbt/adapters/bigquery/impl.py | 14 ++++++++++---- tests/functional/adapter/test_constraints.py | 11 ++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 63ac97135..f3da544c6 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -912,11 +912,17 @@ def python_submission_helpers(self) -> Dict[str, Type[PythonJobHelper]]: @classmethod def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str: - if constraint.type != ConstraintType.not_null: - return "" # Only not null column constraints are supported by BigQuery - else: + if constraint.type == ConstraintType.not_null: return super().render_column_constraint(constraint) + elif constraint.type == ConstraintType.primary_key: + return "primary key not enforced" + else: + return "" @classmethod def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]: - return None # BigQuery doesn't currently support table constraints + if constraint.type == ConstraintType.primary_key: + c = super().render_model_constraint(constraint) + return f"{c} not enforced" if c else None + else: + return None diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 650a8f586..360da8831 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -18,7 +18,7 @@ my_model_wrong_name_sql, my_model_view_wrong_name_sql, my_model_incremental_wrong_name_sql, - model_schema_yml, + model_schema_yml, constrained_model_schema_yml, ) _expected_sql_bigquery = """ @@ -43,8 +43,8 @@ # Different on BigQuery: # - does not support a data type named 'text' (TODO handle this via type translation/aliasing!) # - raises an explicit error, if you try to set a primary key constraint, because it's not enforced -constraints_yml = model_schema_yml.replace("text", "string").replace("primary key", "") - +constraints_yml = model_schema_yml.replace("text", "string") +model_constraints_yml = constrained_model_schema_yml.replace("text", "string") class BigQueryColumnEqualSetup: @pytest.fixture @@ -176,7 +176,7 @@ class TestBigQueryModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntime def models(self): return { "my_model.sql": my_incremental_model_sql, - "constraints_schema.yml": constraints_yml, + "constraints_schema.yml": model_constraints_yml, } @pytest.fixture(scope="class") @@ -185,7 +185,8 @@ def expected_sql(self): create or replace table ( id integer not null, color string, - date_day string + date_day string, + primary key (id) not enforced ) OPTIONS() as ( From da1a7b1f1b1211246a5b76c09345adf805ef65ef Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 6 Apr 2023 17:49:08 -0400 Subject: [PATCH 5/8] CT-2221: Add support for foreign keys as well. --- dbt/adapters/bigquery/impl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index f3da544c6..e8cc29e33 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -916,12 +916,18 @@ def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str: return super().render_column_constraint(constraint) elif constraint.type == ConstraintType.primary_key: return "primary key not enforced" + elif constraint.type == ConstraintType.foreign_key: + c = super().render_column_constraint(constraint) + return f"{c} not enforced" else: return "" @classmethod def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]: - if constraint.type == ConstraintType.primary_key: + if ( + constraint.type == ConstraintType.primary_key + or constraint.type == ConstraintType.foreign_key + ): c = super().render_model_constraint(constraint) return f"{c} not enforced" if c else None else: From 1d4d4712c0b47904fe88335625dae2a747f84bc9 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Mon, 10 Apr 2023 17:14:03 -0400 Subject: [PATCH 6/8] CT-2221: Address review issue. --- dbt/adapters/bigquery/impl.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index e8cc29e33..e7e41a0c3 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -914,9 +914,10 @@ def python_submission_helpers(self) -> Dict[str, Type[PythonJobHelper]]: def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str: if constraint.type == ConstraintType.not_null: return super().render_column_constraint(constraint) - elif constraint.type == ConstraintType.primary_key: - return "primary key not enforced" - elif constraint.type == ConstraintType.foreign_key: + elif ( + constraint.type == ConstraintType.primary_key + or constraint.type == ConstraintType.foreign_key + ): c = super().render_column_constraint(constraint) return f"{c} not enforced" else: From 715f65220ae74ce44dfa784276ef3617f0248a17 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Tue, 11 Apr 2023 12:37:59 -0400 Subject: [PATCH 7/8] CT-2221: Fix some integration test failures --- tests/functional/adapter/test_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 360da8831..27666f064 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -23,7 +23,7 @@ _expected_sql_bigquery = """ create or replace table ( - id integer not null, + id integer not null primary key not enforced, color string, date_day string ) From 46c0ace7a271d8a2559c0f245c019b02232bfe0c Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Tue, 11 Apr 2023 13:26:46 -0400 Subject: [PATCH 8/8] CT-2221: Re-add macro override which should not have been removed. --- dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql new file mode 100644 index 000000000..d078e8ad9 --- /dev/null +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -0,0 +1,5 @@ +{% macro bigquery__format_column(column) -%} + {% set data_type = column.data_type %} + {% set formatted = column.column.lower() ~ " " ~ data_type %} + {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }} +{%- endmacro -%}