[Backport] #165/#188 (#197)

* adding regression test case draft

* backporting #165 to 1.1.0 latest

* trying to take into account CT-604, remove old test, remove BIGQUERY_TEST_DATABASE env

dbt/adapters/bigquery/connections.py

@@ -555,6 +555,9 @@ def table_ref(database, schema, table_name):
     def get_bq_table(self, database, schema, identifier):
         """Get a bigquery table for a schema/model."""
         conn = self.get_thread_connection()
+        # backwards compatibility: fill in with defaults if not specified
+        database = database or conn.credentials.database
+        schema = schema or conn.credentials.schema
         table_ref = self.table_ref(database, schema, identifier)
         return conn.handle.get_table(table_ref)
 

test.env.example

@@ -1,4 +1,3 @@
-BIGQUERY_TEST_DATABASE=
 BIGQUERY_TEST_ALT_DATABASE=
 BIGQUERY_TEST_NO_ACCESS_DATABASE=
 BIGQUERY_TEST_SERVICE_ACCOUNT_JSON='{}'

tests/functional/test_get_columns_incomplete_database.py

@@ -0,0 +1,50 @@
+import pytest
+from dbt.tests.util import run_dbt
+
+# This is to test a edge case found in https://github.com/dbt-labs/dbt-bigquery/pull/165/files
+
+tests__get_cols_in_sql = """
+{% test get_cols_in(model) %}
+
+  {# The step which causes the issue #}
+  {%- set relation = api.Relation.create(identifier=model.table) if execute -%}
+
+  {% set columns = adapter.get_columns_in_relation(relation) %}
+
+  select
+    {% for col in columns %}
+      {{ col.name }} {{ "," if not loop.last }}
+    {% endfor %}
+
+  from {{ model }}
+  limit 0
+
+{% endtest %}
+"""
+
+models__my_model = """select 1 as id, 'text' as another_col
+"""
+
+properties__model_yml = """
+version: 2
+models:
+  - name: my_model
+    tests:
+      - get_cols_in
+"""
+
+class TestIncompleteRelation:
+    @pytest.fixture(scope="class")
+    def properties(self):
+        return {"properties_model_yml.yml": properties__model_yml}
+
+    @pytest.fixture(scope="class")
+    def tests(self):
+        return {"tests__get_col_in.sql": tests__get_cols_in_sql}
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return { "models__my_model.sql": models__my_model }
+
+    def test_incomplete_relation(self, project):
+        run_dbt(["run"])

tests/functional/test_override_database/fixtures.py

@@ -0,0 +1,71 @@
+import pytest
+from dbt.tests.fixtures.project import write_project_files
+
+
+models__view_2_sql = """
+{%- if target.type == 'bigquery' -%}
+  {{ config(project=var('alternate_db')) }}
+{%- else -%}
+  {{ config(database=var('alternate_db')) }}
+{%- endif -%}
+select * from {{ ref('seed') }}
+
+"""
+
+models__view_1_sql = """
+{#
+	We are running against a database that must be quoted.
+	These calls ensure that we trigger an error if we're failing to quote at parse-time
+#}
+{% do adapter.already_exists(this.schema, this.table) %}
+{% do adapter.get_relation(this.database, this.schema, this.table) %}
+select * from {{ ref('seed') }}
+
+"""
+
+models__subfolder__view_4_sql = """
+{{
+    config(database=var('alternate_db'))
+}}
+
+select * from {{ ref('seed') }}
+
+"""
+
+models__subfolder__view_3_sql = """
+select * from {{ ref('seed') }}
+
+"""
+
+seeds__seed_csv = """id,name
+1,a
+2,b
+3,c
+4,d
+5,e
+"""
+
+@pytest.fixture(scope="class")
+def models():
+    return {
+      "view_2.sql": models__view_2_sql,
+      "view_1.sql": models__view_1_sql,
+      "subfolder":
+        {
+          "view_4.sql": models__subfolder__view_4_sql,
+          "view_3.sql": models__subfolder__view_3_sql,
+          },
+    }
+
+@pytest.fixture(scope="class")
+def seeds():
+    return {
+      "seed.csv": seeds__seed_csv
+      }
+
+@pytest.fixture(scope="class")
+def project_files(project_root, models, seeds,):
+    write_project_files(project_root, "models", models)
+    write_project_files(project_root, "seeds", seeds)
+
+

tests/functional/test_override_database/test_override_database.py

@@ -0,0 +1,158 @@
+import pytest
+from dbt.tests.util import run_dbt, check_relations_equal, check_relations_equal_with_relations
+from tests.functional.test_override_database.fixtures import (
+    models,
+    seeds,
+    project_files
+)
+import os
+
+
+
+
+class BaseOverrideDatabase:
+    @pytest.fixture(scope="class")
+    def model_path(self):
+        return "models"
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "config-version": 2,
+            "seed-paths": ["seeds"],
+            "vars": {
+                "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+            },
+            "quoting": {
+                "database": True,
+            },
+            "seeds": {
+                "quote_columns": False,
+            }
+        }
+
+
+class TestModelOverrideBigQuery(BaseOverrideDatabase):
+    def run_database_override(self, project):
+        run_dbt(["seed"])
+        assert len(run_dbt(["run"])) == 4
+        check_relations_equal_with_relations(project.adapter, [
+            project.adapter.Relation.create(schema=project.test_schema, identifier="seed"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"),
+            project.adapter.Relation.create(schema=project.test_schema, identifier="view_1"),
+            project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4")
+        ])
+
+
+    def test_bigquery_database_override(self, project):
+        self.run_database_override(project)
+
+
+class BaseTestProjectModelOverrideBigQuery(BaseOverrideDatabase):
+
+    def run_database_override(self, project):
+        run_dbt(["seed"])
+        assert len(run_dbt(["run"])) == 4
+        self.assertExpectedRelations(project)
+
+    def assertExpectedRelations(self, project):
+        check_relations_equal_with_relations(project.adapter, [
+            project.adapter.Relation.create(schema=project.test_schema, identifier="seed"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_1"),
+            project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4")
+        ])
+
+
+class TestProjectModelOverrideBigQuery(BaseTestProjectModelOverrideBigQuery):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "config-version": 2,
+            "vars": {
+                "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+            },
+            "models": {
+                "database": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+                "test": {
+                    "subfolder": {
+                        "database": "{{ target.database }}"
+                    }
+                }
+            },
+            "seed-paths": ["seeds"],
+            "vars": {
+                "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+            },
+            "quoting": {
+                "database": True,
+            },
+            "seeds": {
+                "quote_columns": False,
+            }
+        }
+
+    def test_bigquery_database_override(self, project):
+        self.run_database_override(project)
+
+
+class TestProjectModelAliasOverrideBigQuery(BaseTestProjectModelOverrideBigQuery):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "config-version": 2,
+            "vars": {
+                "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+            },
+            "models": {
+                "project": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+                "test": {
+                    "subfolder": {
+                        "project": "{{ target.database }}"
+                    }
+                }
+            },
+            "seed-paths": ["seeds"],
+            "vars": {
+                "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+            },
+            "quoting": {
+                "database": True,
+            },
+            "seeds": {
+                "quote_columns": False,
+            }
+        }
+
+    def test_bigquery_project_override(self, project):
+        self.run_database_override(project)
+
+
+class TestProjectSeedOverrideBigQuery(BaseOverrideDatabase):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "config-version": 2,
+            "seed-paths": ["seeds"],
+            "vars": {
+                "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"),
+            },
+            "seeds": {
+                "database": os.getenv("BIGQUERY_TEST_ALT_DATABASE")
+            }
+        }
+    def run_database_override(self, project):
+        run_dbt(["seed"])
+        assert len(run_dbt(["run"])) == 4
+        check_relations_equal_with_relations(project.adapter, [
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="seed"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"),
+            project.adapter.Relation.create(schema=project.test_schema, identifier="view_1"),
+            project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"),
+            project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4")
+        ])
+
+    def test_bigquery_database_override(self, project):
+        self.run_database_override(project)