diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index a445796b5..454e84d7d 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -555,6 +555,9 @@ def table_ref(database, schema, table_name): def get_bq_table(self, database, schema, identifier): """Get a bigquery table for a schema/model.""" conn = self.get_thread_connection() + # backwards compatibility: fill in with defaults if not specified + database = database or conn.credentials.database + schema = schema or conn.credentials.schema table_ref = self.table_ref(database, schema, identifier) return conn.handle.get_table(table_ref) diff --git a/test.env.example b/test.env.example index 031968c60..2065e4393 100644 --- a/test.env.example +++ b/test.env.example @@ -1,4 +1,3 @@ -BIGQUERY_TEST_DATABASE= BIGQUERY_TEST_ALT_DATABASE= BIGQUERY_TEST_NO_ACCESS_DATABASE= BIGQUERY_TEST_SERVICE_ACCOUNT_JSON='{}' diff --git a/tests/functional/test_get_columns_incomplete_database.py b/tests/functional/test_get_columns_incomplete_database.py new file mode 100644 index 000000000..55aab045c --- /dev/null +++ b/tests/functional/test_get_columns_incomplete_database.py @@ -0,0 +1,50 @@ +import pytest +from dbt.tests.util import run_dbt + +# This is to test a edge case found in https://github.com/dbt-labs/dbt-bigquery/pull/165/files + +tests__get_cols_in_sql = """ +{% test get_cols_in(model) %} + + {# The step which causes the issue #} + {%- set relation = api.Relation.create(identifier=model.table) if execute -%} + + {% set columns = adapter.get_columns_in_relation(relation) %} + + select + {% for col in columns %} + {{ col.name }} {{ "," if not loop.last }} + {% endfor %} + + from {{ model }} + limit 0 + +{% endtest %} +""" + +models__my_model = """select 1 as id, 'text' as another_col +""" + +properties__model_yml = """ +version: 2 +models: + - name: my_model + tests: + - get_cols_in +""" + +class TestIncompleteRelation: + @pytest.fixture(scope="class") + def properties(self): + return {"properties_model_yml.yml": properties__model_yml} + + @pytest.fixture(scope="class") + def tests(self): + return {"tests__get_col_in.sql": tests__get_cols_in_sql} + + @pytest.fixture(scope="class") + def models(self): + return { "models__my_model.sql": models__my_model } + + def test_incomplete_relation(self, project): + run_dbt(["run"]) \ No newline at end of file diff --git a/tests/functional/test_override_database/fixtures.py b/tests/functional/test_override_database/fixtures.py new file mode 100644 index 000000000..315a74e8f --- /dev/null +++ b/tests/functional/test_override_database/fixtures.py @@ -0,0 +1,71 @@ +import pytest +from dbt.tests.fixtures.project import write_project_files + + +models__view_2_sql = """ +{%- if target.type == 'bigquery' -%} + {{ config(project=var('alternate_db')) }} +{%- else -%} + {{ config(database=var('alternate_db')) }} +{%- endif -%} +select * from {{ ref('seed') }} + +""" + +models__view_1_sql = """ +{# + We are running against a database that must be quoted. + These calls ensure that we trigger an error if we're failing to quote at parse-time +#} +{% do adapter.already_exists(this.schema, this.table) %} +{% do adapter.get_relation(this.database, this.schema, this.table) %} +select * from {{ ref('seed') }} + +""" + +models__subfolder__view_4_sql = """ +{{ + config(database=var('alternate_db')) +}} + +select * from {{ ref('seed') }} + +""" + +models__subfolder__view_3_sql = """ +select * from {{ ref('seed') }} + +""" + +seeds__seed_csv = """id,name +1,a +2,b +3,c +4,d +5,e +""" + +@pytest.fixture(scope="class") +def models(): + return { + "view_2.sql": models__view_2_sql, + "view_1.sql": models__view_1_sql, + "subfolder": + { + "view_4.sql": models__subfolder__view_4_sql, + "view_3.sql": models__subfolder__view_3_sql, + }, + } + +@pytest.fixture(scope="class") +def seeds(): + return { + "seed.csv": seeds__seed_csv + } + +@pytest.fixture(scope="class") +def project_files(project_root, models, seeds,): + write_project_files(project_root, "models", models) + write_project_files(project_root, "seeds", seeds) + + diff --git a/tests/functional/test_override_database/test_override_database.py b/tests/functional/test_override_database/test_override_database.py new file mode 100644 index 000000000..8ce179056 --- /dev/null +++ b/tests/functional/test_override_database/test_override_database.py @@ -0,0 +1,158 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal, check_relations_equal_with_relations +from tests.functional.test_override_database.fixtures import ( + models, + seeds, + project_files +) +import os + + + + +class BaseOverrideDatabase: + @pytest.fixture(scope="class") + def model_path(self): + return "models" + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "quoting": { + "database": True, + }, + "seeds": { + "quote_columns": False, + } + } + + +class TestModelOverrideBigQuery(BaseOverrideDatabase): + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + check_relations_equal_with_relations(project.adapter, [ + project.adapter.Relation.create(schema=project.test_schema, identifier="seed"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_1"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4") + ]) + + + def test_bigquery_database_override(self, project): + self.run_database_override(project) + + +class BaseTestProjectModelOverrideBigQuery(BaseOverrideDatabase): + + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + self.assertExpectedRelations(project) + + def assertExpectedRelations(self, project): + check_relations_equal_with_relations(project.adapter, [ + project.adapter.Relation.create(schema=project.test_schema, identifier="seed"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_1"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4") + ]) + + +class TestProjectModelOverrideBigQuery(BaseTestProjectModelOverrideBigQuery): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "models": { + "database": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + "test": { + "subfolder": { + "database": "{{ target.database }}" + } + } + }, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "quoting": { + "database": True, + }, + "seeds": { + "quote_columns": False, + } + } + + def test_bigquery_database_override(self, project): + self.run_database_override(project) + + +class TestProjectModelAliasOverrideBigQuery(BaseTestProjectModelOverrideBigQuery): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "models": { + "project": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + "test": { + "subfolder": { + "project": "{{ target.database }}" + } + } + }, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "quoting": { + "database": True, + }, + "seeds": { + "quote_columns": False, + } + } + + def test_bigquery_project_override(self, project): + self.run_database_override(project) + + +class TestProjectSeedOverrideBigQuery(BaseOverrideDatabase): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "seeds": { + "database": os.getenv("BIGQUERY_TEST_ALT_DATABASE") + } + } + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + check_relations_equal_with_relations(project.adapter, [ + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="seed"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_1"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4") + ]) + + def test_bigquery_database_override(self, project): + self.run_database_override(project) diff --git a/tests/integration/override_database_test/models/subfolder/view_3.sql b/tests/integration/override_database_test/models/subfolder/view_3.sql deleted file mode 100644 index 4b91aa0f2..000000000 --- a/tests/integration/override_database_test/models/subfolder/view_3.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/models/subfolder/view_4.sql b/tests/integration/override_database_test/models/subfolder/view_4.sql deleted file mode 100644 index efa1268fa..000000000 --- a/tests/integration/override_database_test/models/subfolder/view_4.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ - config(database=var('alternate_db')) -}} - -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/models/view_1.sql b/tests/integration/override_database_test/models/view_1.sql deleted file mode 100644 index a43f04646..000000000 --- a/tests/integration/override_database_test/models/view_1.sql +++ /dev/null @@ -1,7 +0,0 @@ -{# - We are running against a database that must be quoted. - These calls ensure that we trigger an error if we're failing to quote at parse-time -#} -{% do adapter.already_exists(this.schema, this.table) %} -{% do adapter.get_relation(this.database, this.schema, this.table) %} -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/models/view_2.sql b/tests/integration/override_database_test/models/view_2.sql deleted file mode 100644 index 9ac6bdad6..000000000 --- a/tests/integration/override_database_test/models/view_2.sql +++ /dev/null @@ -1,6 +0,0 @@ -{%- if target.type == 'bigquery' -%} - {{ config(project=var('alternate_db')) }} -{%- else -%} - {{ config(database=var('alternate_db')) }} -{%- endif -%} -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/seeds/seed.csv b/tests/integration/override_database_test/seeds/seed.csv deleted file mode 100644 index ae9125c16..000000000 --- a/tests/integration/override_database_test/seeds/seed.csv +++ /dev/null @@ -1,6 +0,0 @@ -id,name -1,a -2,b -3,c -4,d -5,e diff --git a/tests/integration/override_database_test/test_override_database.py b/tests/integration/override_database_test/test_override_database.py deleted file mode 100644 index 9b12b3e79..000000000 --- a/tests/integration/override_database_test/test_override_database.py +++ /dev/null @@ -1,176 +0,0 @@ -from tests.integration.base import DBTIntegrationTest, use_profile - -import os - - -class BaseOverrideDatabase(DBTIntegrationTest): - setup_alternate_db = True - @property - def schema(self): - return "override_database" - - @property - def models(self): - return "models" - - @property - def alternative_database(self): - return super().alternative_database - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - -class TestModelOverride(BaseOverrideDatabase): - def run_database_override(self): - func = lambda x: x - - self.run_dbt(['seed']) - - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.default_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - -class BaseTestProjectModelOverride(BaseOverrideDatabase): - # this is janky, but I really want to access self.default_database in - # project_config - @property - def default_database(self): - target = self._profile_config['test']['target'] - profile = self._profile_config['test']['outputs'][target] - for key in ['database', 'project', 'dbname']: - if key in profile: - database = profile[key] - return database - assert False, 'No profile database found!' - - def run_database_override(self): - self.run_dbt(['seed']) - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertExpectedRelations() - - def assertExpectedRelations(self): - func = lambda x: x - - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.alternative_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - -class TestProjectModelOverride(BaseTestProjectModelOverride): - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'models': { - 'database': self.alternative_database, - 'test': { - 'subfolder': { - 'database': self.default_database, - } - } - }, - 'seed-paths': ['seeds'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - -class TestProjectModelAliasOverride(BaseTestProjectModelOverride): - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'models': { - 'project': self.alternative_database, - 'test': { - 'subfolder': { - 'project': self.default_database, - } - } - }, - 'seed-paths': ['seeds'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('bigquery') - def test_bigquery_project_override(self): - self.run_database_override() - - -class TestProjectSeedOverride(BaseOverrideDatabase): - def run_database_override(self): - func = lambda x: x - - self.use_default_project({ - 'config-version': 2, - 'seeds': { - 'database': self.alternative_database - }, - }) - self.run_dbt(['seed']) - - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.alternative_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.default_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override()