diff --git a/Makefile b/Makefile index 3990df87e57..c5c2aba59a4 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,6 @@ test-integration: @echo "Integration test run starting..." @time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py36,integration-snowflake-py27,integration-snowflake-py36,integration-bigquery-py27,integration-bigquery-py36 - test-quick: @echo "Integration test run starting..." @time docker-compose run test tox -e integration-postgres-py36 -- -x diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 5c8f130c71e..f0bb48c4309 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -89,7 +89,7 @@ def create_from_node(cls, profile, node, **kwargs): return cls.create( project=profile.get('project'), schema=node.get('schema'), - identifier=node.get('name'), + identifier=node.get('alias'), **kwargs) @classmethod diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 869a022a856..686b0a003f4 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -177,7 +177,7 @@ def create_from_node(cls, profile, node, table_name=None, **kwargs): return cls.create( database=profile.get('dbname'), schema=node.get('schema'), - identifier=node.get('name'), + identifier=node.get('alias'), table_name=table_name, **kwargs) diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index d8765c54775..11dccd02223 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -48,5 +48,5 @@ def create_from_node(cls, profile, node, **kwargs): return cls.create( database=profile.get('database'), schema=node.get('schema'), - identifier=node.get('name'), + identifier=node.get('alias'), **kwargs) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 8d44c8736bb..5fe52920472 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -1,5 +1,5 @@ -from voluptuous import Schema, Required, All, Any, Length, ALLOW_EXTRA -from voluptuous import Optional +from voluptuous import Schema, Required, Optional, All, Any, Length +from voluptuous import ALLOW_EXTRA import dbt.exceptions @@ -34,6 +34,7 @@ Required('unique_id'): All(basestring, Length(min=1, max=255)), Required('fqn'): All(list, [All(basestring)]), Required('schema'): basestring, + Optional('alias'): basestring, Required('refs'): [All(tuple)], diff --git a/dbt/exceptions.py b/dbt/exceptions.py index fa2fdb2d7fa..ac5fbf7c760 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -164,6 +164,10 @@ def ref_bad_context(model, target_model_name, target_model_package): To fix this, add the following hint to the top of the model "{model_name}": -- depends_on: {ref_string}""" + # This explicitly references model['name'], instead of model['alias'], for + # better error messages. Ex. If models foo_users and bar_users are aliased + # to 'users', in their respective schemas, then you would want to see + # 'bar_users' in your error messge instead of just 'users'. error_msg = base_error_msg.format( model_name=model['name'], model_path=model['path'], diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql index 8ac99e9ae7c..9be4edd6265 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -127,8 +127,8 @@ {% endcall %} {% endfor %} - {%- set identifier = model['name'] -%} - {%- set tmp_identifier = model['name'] + '__dbt_archival_tmp' -%} + {%- set identifier = model['alias'] -%} + {%- set tmp_identifier = identifier + '__dbt_archival_tmp' -%} {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='table') -%} {% call statement() %} diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index 71945f29f08..842500818e0 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -1,7 +1,7 @@ {% macro dbt__incremental_delete(target_relation, tmp_relation) -%} {%- set unique_key = config.require('unique_key') -%} - {%- set identifier = model['name'] -%} + {%- set identifier = model['alias'] -%} delete from {{ target_relation }} @@ -16,8 +16,8 @@ {%- set sql_where = config.require('sql_where') -%} {%- set unique_key = config.get('unique_key') -%} - {%- set identifier = model['name'] -%} - {%- set tmp_identifier = model['name'] + '__dbt_incremental_tmp' -%} + {%- set identifier = model['alias'] -%} + {%- set tmp_identifier = identifier + '__dbt_incremental_tmp' -%} {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, diff --git a/dbt/include/global_project/macros/materializations/seed/bigquery.sql b/dbt/include/global_project/macros/materializations/seed/bigquery.sql index ec79c070cb2..11f67a9dbab 100644 --- a/dbt/include/global_project/macros/materializations/seed/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/seed/bigquery.sql @@ -10,6 +10,6 @@ {% macro bigquery__load_csv_rows(model) %} {%- set column_override = model['config'].get('column_types', {}) -%} - {{ adapter.load_dataframe(model['schema'], model['name'], model['agate_table'], column_override) }} + {{ adapter.load_dataframe(model['schema'], model['alias'], model['agate_table'], column_override) }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/dbt/include/global_project/macros/materializations/seed/seed.sql index f0a8ca86f95..08c06d7cde1 100644 --- a/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/dbt/include/global_project/macros/materializations/seed/seed.sql @@ -86,7 +86,7 @@ {% materialization seed, default %} - {%- set identifier = model['name'] -%} + {%- set identifier = model['alias'] -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} {%- set existing_relations = adapter.list_relations(schema=schema) -%} diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 3c50969bbab..837a1de0456 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -28,7 +28,7 @@ {% materialization table, adapter='bigquery' -%} - {%- set identifier = model['name'] -%} + {%- set identifier = model['alias'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index f03d8b46d8d..1610b9b6c9c 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -1,5 +1,5 @@ {% materialization table, default %} - {%- set identifier = model['name'] -%} + {%- set identifier = model['alias'] -%} {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} diff --git a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql index 1ab47ea9c68..338c5ccfdcd 100644 --- a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql +++ b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql @@ -1,6 +1,6 @@ {% materialization view, adapter='bigquery' -%} - {%- set identifier = model['name'] -%} + {%- set identifier = model['alias'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing_relations = adapter.list_relations(schema=schema) -%} diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/dbt/include/global_project/macros/materializations/view/view.sql index 1f0036116bc..56ba2e4577c 100644 --- a/dbt/include/global_project/macros/materializations/view/view.sql +++ b/dbt/include/global_project/macros/materializations/view/view.sql @@ -1,6 +1,6 @@ {%- materialization view, default -%} - {%- set identifier = model['name'] -%} + {%- set identifier = model['alias'] -%} {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} diff --git a/dbt/model.py b/dbt/model.py index e169f25007f..f96676b3ea3 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -14,6 +14,7 @@ class SourceConfig(object): AppendListFields = ['pre-hook', 'post-hook'] ExtendDictFields = ['vars', 'column_types', 'quoting'] ClobberFields = [ + 'alias', 'schema', 'enabled', 'materialized', diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 7da8080c6ce..2ea13c993e9 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -251,17 +251,17 @@ def _node_context(cls, adapter, project, node): def call_get_columns_in_table(schema_name, table_name): return adapter.get_columns_in_table( profile, project, schema_name, - table_name, model_name=node.get('name')) + table_name, model_name=node.get('alias')) def call_get_missing_columns(from_schema, from_table, to_schema, to_table): return adapter.get_missing_columns( profile, project, from_schema, from_table, - to_schema, to_table, node.get('name')) + to_schema, to_table, node.get('alias')) def call_already_exists(schema, table): return adapter.already_exists( - profile, project, schema, table, node.get('name')) + profile, project, schema, table, node.get('alias')) return { "run_started_at": dbt.tracking.active_user.run_started_at, @@ -385,8 +385,7 @@ def after_hooks(cls, project, adapter, results, flat_graph, elapsed): def describe_node(self): materialization = dbt.utils.get_materialization(self.node) schema_name = self.node.get('schema') - node_name = self.node.get('name') - + node_name = dbt.utils.get_alias(self.node) return "{} model {}.{}".format(materialization, schema_name, node_name) def print_start_line(self): diff --git a/dbt/parser.py b/dbt/parser.py index 60464b7997c..2ed94032fd0 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -223,7 +223,7 @@ def parse_node(node, node_path, root_project_config, package_project_config, profile = dbt.utils.get_profile_from_project(root_project_config) default_schema = profile.get('schema', 'public') node['schema'] = default_schema - + node['alias'] = dbt.utils.get_alias(node) context = dbt.context.parser.generate(node, root_project_config, {"macros": macros}) diff --git a/dbt/ui/printer.py b/dbt/ui/printer.py index 871bedd2c63..5c9bc843cff 100644 --- a/dbt/ui/printer.py +++ b/dbt/ui/printer.py @@ -156,7 +156,7 @@ def print_model_result_line(result, schema_name, index, total): info=info, model_type=get_materialization(model), schema=schema_name, - relation=model.get('name')), + relation=model.get('alias')), status, index, total, @@ -187,7 +187,7 @@ def print_seed_result_line(result, schema_name, index, total): "{info} seed file {schema}.{relation}".format( info=info, schema=schema_name, - relation=model.get('name')), + relation=model.get('alias')), status, index, total, diff --git a/dbt/utils.py b/dbt/utils.py index 9b71b7c18a3..4008c33ab00 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -15,6 +15,7 @@ DBTConfigKeys = [ + 'alias', 'schema', 'enabled', 'materialized', @@ -64,7 +65,7 @@ def get_model_name_or_none(model): elif isinstance(model, basestring): name = model elif isinstance(model, dict): - name = model.get('name') + name = get_alias(model) else: name = model.nice_name return name @@ -86,7 +87,7 @@ def model_immediate_name(model, non_destructive): seeds. """ - model_name = model.get('name') + model_name = get_alias(model) is_incremental = (get_materialization(model) == 'incremental') is_seed = is_type(model, 'seed') @@ -298,6 +299,10 @@ def get_materialization(node): return node.get('config', {}).get('materialized') +def get_alias(node): + return node.get('config', {}).get('alias', node.get('name')) + + def is_enabled(node): return node.get('config', {}).get('enabled') is True diff --git a/test/integration/026_aliases_test/models/foo_alias.sql b/test/integration/026_aliases_test/models/foo_alias.sql new file mode 100644 index 00000000000..358eb8032fe --- /dev/null +++ b/test/integration/026_aliases_test/models/foo_alias.sql @@ -0,0 +1,10 @@ + +{{ + config( + alias='foo', + materialized='table' + ) +}} + +SELECT + '{{ this.alias }}' as "tablename" diff --git a/test/integration/026_aliases_test/models/ref_foo_alias.sql b/test/integration/026_aliases_test/models/ref_foo_alias.sql new file mode 100644 index 00000000000..2839e613fb3 --- /dev/null +++ b/test/integration/026_aliases_test/models/ref_foo_alias.sql @@ -0,0 +1,18 @@ + +{{ + config( + materialized='table' + ) +}} + +WITH trigger_ref AS ( + SELECT + * + FROM + -- we should still be able to ref a model by its filepath + {{ ref('foo_alias') }} +) + +SELECT + -- this name should still be the filename + '{{ this.alias }}' as "tablename" \ No newline at end of file diff --git a/test/integration/026_aliases_test/test_aliases.py b/test/integration/026_aliases_test/test_aliases.py new file mode 100644 index 00000000000..a6a4b5488e0 --- /dev/null +++ b/test/integration/026_aliases_test/test_aliases.py @@ -0,0 +1,60 @@ +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest + + +class TestAliases(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + + @property + def schema(self): + return "aliases_026" + + @property + def models(self): + return "test/integration/026_aliases_test/models" + + @property + def profile_config(self): + return { + 'test': { + 'outputs': { + 'dev': { + 'type': 'postgres', + 'threads': 1, + 'host': 'database', + 'port': 5432, + 'user': "root", + 'pass': "password", + 'dbname': 'dbt', + 'schema': self.unique_schema() + }, + }, + 'target': 'dev' + } + } + + @property + def query_foo_alias(self): + return """ + select + tablename + from {schema}.foo + """.format(schema=self.unique_schema()) + + @property + def query_ref_foo_alias(self): + return """ + select + tablename + from {schema}.ref_foo_alias + """.format(schema=self.unique_schema()) + + @attr(type='postgres') + def test__alias_model_name(self): + self.run_dbt(['run']) + result = self.run_sql(self.query_foo_alias, fetch='all')[0][0] + self.assertEqual(result, 'foo') + result = self.run_sql(self.query_ref_foo_alias, fetch='all')[0][0] + self.assertEqual(result, 'ref_foo_alias') diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 570c48c4ba0..063d56b3b1f 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -92,6 +92,7 @@ def test__single_model(self): 'snowplow': self.snowplow_project_config}), { 'model.root.model_one': { + 'alias': 'model_one', 'name': 'model_one', 'schema': 'analytics', 'resource_type': 'model', @@ -150,6 +151,7 @@ def test__single_model__nested_configuration(self): 'snowplow': self.snowplow_project_config}), { 'model.root.model_one': { + 'alias': 'model_one', 'name': 'model_one', 'schema': 'analytics', 'resource_type': 'model', @@ -191,6 +193,7 @@ def test__empty_model(self): {'root': self.root_project_config}), { 'model.root.model_one': { + 'alias': 'model_one', 'name': 'model_one', 'schema': 'analytics', 'resource_type': 'model', @@ -241,6 +244,7 @@ def test__simple_dependency(self): 'snowplow': self.snowplow_project_config}), { 'model.root.base': { + 'alias': 'base', 'name': 'base', 'schema': 'analytics', 'resource_type': 'model', @@ -262,6 +266,7 @@ def test__simple_dependency(self): models, 'base').get('raw_sql') }, 'model.root.events_tx': { + 'alias': 'events_tx', 'name': 'events_tx', 'schema': 'analytics', 'resource_type': 'model', @@ -340,6 +345,7 @@ def test__multiple_dependencies(self): 'snowplow': self.snowplow_project_config}), { 'model.root.events': { + 'alias': 'events', 'name': 'events', 'schema': 'analytics', 'resource_type': 'model', @@ -361,6 +367,7 @@ def test__multiple_dependencies(self): models, 'events').get('raw_sql') }, 'model.root.sessions': { + 'alias': 'sessions', 'name': 'sessions', 'schema': 'analytics', 'resource_type': 'model', @@ -382,6 +389,7 @@ def test__multiple_dependencies(self): models, 'sessions').get('raw_sql') }, 'model.root.events_tx': { + 'alias': 'events_tx', 'name': 'events_tx', 'schema': 'analytics', 'resource_type': 'model', @@ -403,6 +411,7 @@ def test__multiple_dependencies(self): models, 'events_tx').get('raw_sql') }, 'model.root.sessions_tx': { + 'alias': 'sessions_tx', 'name': 'sessions_tx', 'schema': 'analytics', 'resource_type': 'model', @@ -424,6 +433,7 @@ def test__multiple_dependencies(self): models, 'sessions_tx').get('raw_sql') }, 'model.root.multi': { + 'alias': 'multi', 'name': 'multi', 'schema': 'analytics', 'resource_type': 'model', @@ -504,6 +514,7 @@ def test__multiple_dependencies__packages(self): 'snowplow': self.snowplow_project_config}), { 'model.snowplow.events': { + 'alias': 'events', 'name': 'events', 'schema': 'analytics', 'resource_type': 'model', @@ -525,6 +536,7 @@ def test__multiple_dependencies__packages(self): models, 'events').get('raw_sql') }, 'model.snowplow.sessions': { + 'alias': 'sessions', 'name': 'sessions', 'schema': 'analytics', 'resource_type': 'model', @@ -546,6 +558,7 @@ def test__multiple_dependencies__packages(self): models, 'sessions').get('raw_sql') }, 'model.snowplow.events_tx': { + 'alias': 'events_tx', 'name': 'events_tx', 'schema': 'analytics', 'resource_type': 'model', @@ -567,6 +580,7 @@ def test__multiple_dependencies__packages(self): models, 'events_tx').get('raw_sql') }, 'model.snowplow.sessions_tx': { + 'alias': 'sessions_tx', 'name': 'sessions_tx', 'schema': 'analytics', 'resource_type': 'model', @@ -588,6 +602,7 @@ def test__multiple_dependencies__packages(self): models, 'sessions_tx').get('raw_sql') }, 'model.root.multi': { + 'alias': 'multi', 'name': 'multi', 'schema': 'analytics', 'resource_type': 'model', @@ -772,6 +787,7 @@ def test__in_model_config(self): 'snowplow': self.snowplow_project_config}), { 'model.root.model_one': { + 'alias': 'model_one', 'name': 'model_one', 'schema': 'analytics', 'resource_type': 'model', @@ -854,6 +870,7 @@ def test__root_project_config(self): 'snowplow': self.snowplow_project_config}), { 'model.root.table': { + 'alias': 'table', 'name': 'table', 'schema': 'analytics', 'resource_type': 'model', @@ -875,6 +892,7 @@ def test__root_project_config(self): models, 'table').get('raw_sql') }, 'model.root.ephemeral': { + 'alias': 'ephemeral', 'name': 'ephemeral', 'schema': 'analytics', 'resource_type': 'model', @@ -896,6 +914,7 @@ def test__root_project_config(self): models, 'ephemeral').get('raw_sql') }, 'model.root.view': { + 'alias': 'view', 'name': 'view', 'schema': 'analytics', 'resource_type': 'model', @@ -1045,6 +1064,7 @@ def test__other_project_config(self): 'snowplow': self.snowplow_project_config}), { 'model.root.table': { + 'alias': 'table', 'name': 'table', 'schema': 'analytics', 'resource_type': 'model', @@ -1066,6 +1086,7 @@ def test__other_project_config(self): models, 'table').get('raw_sql') }, 'model.root.ephemeral': { + 'alias': 'ephemeral', 'name': 'ephemeral', 'schema': 'analytics', 'resource_type': 'model', @@ -1087,6 +1108,7 @@ def test__other_project_config(self): models, 'ephemeral').get('raw_sql') }, 'model.root.view': { + 'alias': 'view', 'name': 'view', 'schema': 'analytics', 'resource_type': 'model', @@ -1108,6 +1130,7 @@ def test__other_project_config(self): models, 'view').get('raw_sql') }, 'model.snowplow.multi_sort': { + 'alias': 'multi_sort', 'name': 'multi_sort', 'schema': 'analytics', 'resource_type': 'model', @@ -1160,6 +1183,7 @@ def test__simple_schema_test(self): 'snowplow': self.snowplow_project_config}), { 'test.root.not_null_model_one_id': { + 'alias': 'not_null_model_one_id', 'name': 'not_null_model_one_id', 'schema': 'analytics', 'resource_type': 'test', @@ -1181,6 +1205,7 @@ def test__simple_schema_test(self): 'raw_sql': not_null_sql, }, 'test.root.unique_model_one_id': { + 'alias': 'unique_model_one_id', 'name': 'unique_model_one_id', 'schema': 'analytics', 'resource_type': 'test', @@ -1201,6 +1226,7 @@ def test__simple_schema_test(self): 'raw_sql': unique_sql, }, 'test.root.accepted_values_model_one_id__a__b': { + 'alias': 'accepted_values_model_one_id__a__b', 'name': 'accepted_values_model_one_id__a__b', 'schema': 'analytics', 'resource_type': 'test', @@ -1223,6 +1249,7 @@ def test__simple_schema_test(self): 'raw_sql': accepted_values_sql, }, 'test.root.relationships_model_one_id__id__ref_model_two_': { + 'alias': 'relationships_model_one_id__id__ref_model_two_', 'name': 'relationships_model_one_id__id__ref_model_two_', 'schema': 'analytics', 'resource_type': 'test', @@ -1315,6 +1342,7 @@ def test__simple_data_test(self): 'snowplow': self.snowplow_project_config}), { 'test.root.no_events': { + 'alias': 'no_events', 'name': 'no_events', 'schema': 'analytics', 'resource_type': 'test', @@ -1430,6 +1458,7 @@ def test__simple_macro_used_in_model(self): 'snowplow': self.snowplow_project_config}), { 'model.root.model_one': { + 'alias': 'model_one', 'name': 'model_one', 'schema': 'analytics', 'resource_type': 'model', @@ -1472,6 +1501,7 @@ def test__macro_no_explicit_project_used_in_model(self): 'snowplow': self.snowplow_project_config}), { 'model.root.model_one': { + 'alias': 'model_one', 'name': 'model_one', 'schema': 'analytics', 'resource_type': 'model',