Reorg to align with dbt-core: Part II (#528)

dbt/include/databricks/macros/catalog.sql → dbt/include/databricks/macros/adapters/catalog.sql

@@ -1,24 +1,3 @@
-{% macro current_catalog() -%}
-  {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}
-{% endmacro %}
-
-{% macro databricks__current_catalog() -%}
-  {% call statement('current_catalog', fetch_result=True) %}
-      select current_catalog()
-  {% endcall %}
-  {% do return(load_result('current_catalog').table) %}
-{% endmacro %}
-
-{% macro use_catalog(catalog) -%}
-  {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}
-{% endmacro %}
-
-{% macro databricks__use_catalog(catalog) -%}
-  {% call statement() %}
-    use catalog {{ adapter.quote(catalog) }}
-  {% endcall %}
-{% endmacro %}
-
 {% macro get_catalog(information_schema, schemas) -%}
   {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}
 {% endmacro %}

dbt/include/databricks/macros/adapters/databricks_catalog.sql

@@ -0,0 +1,20 @@
+{% macro current_catalog() -%}
+  {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}
+{% endmacro %}
+
+{% macro databricks__current_catalog() -%}
+  {% call statement('current_catalog', fetch_result=True) %}
+      select current_catalog()
+  {% endcall %}
+  {% do return(load_result('current_catalog').table) %}
+{% endmacro %}
+
+{% macro use_catalog(catalog) -%}
+  {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}
+{% endmacro %}
+
+{% macro databricks__use_catalog(catalog) -%}
+  {% call statement() %}
+    use catalog {{ adapter.quote(catalog) }}
+  {% endcall %}
+{% endmacro %}

dbt/include/databricks/macros/adapters/metadata.sql

@@ -0,0 +1,72 @@
+{% macro databricks__list_relations_without_caching(schema_relation) %}
+  {{ return(adapter.get_relations_without_caching(schema_relation)) }}
+{% endmacro %}
+
+{% macro show_table_extended(schema_relation) %}
+  {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}
+{% endmacro %}
+
+{% macro databricks__show_table_extended(schema_relation) %}
+  {% call statement('show_table_extended', fetch_result=True) -%}
+    show table extended in {{ schema_relation.without_identifier() }} like '{{ schema_relation.identifier }}'
+  {% endcall %}
+
+  {% do return(load_result('show_table_extended').table) %}
+{% endmacro %}
+
+{% macro show_tables(relation) %}
+  {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}
+{% endmacro %}
+
+{% macro databricks__show_tables(relation) %}
+  {% call statement('show_tables', fetch_result=True) -%}
+    show tables in {{ relation }}
+  {% endcall %}
+
+  {% do return(load_result('show_tables').table) %}
+{% endmacro %}
+
+{% macro show_views(relation) %}
+  {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}
+{% endmacro %}
+
+{% macro databricks__show_views(relation) %}
+  {% call statement('show_views', fetch_result=True) -%}
+    show views in {{ relation }}
+  {% endcall %}
+
+  {% do return(load_result('show_views').table) %}
+{% endmacro %}
+
+{% macro databricks__get_relation_last_modified(information_schema, relations) -%}
+
+  {%- call statement('last_modified', fetch_result=True) -%}
+    {% if information_schema.is_hive_metastore %}
+        {%- for relation in relations -%}
+            select '{{ relation.schema }}' as schema,
+                    '{{ relation.identifier }}' as identifier,
+                    max(timestamp) as last_modified,
+                    {{ current_timestamp() }} as snapshotted_at
+            from (describe history {{ relation.schema }}.{{ relation.identifier }})
+            {% if not loop.last %}
+            union all
+            {% endif %}
+        {%- endfor -%}
+    {% else %}
+        select table_schema as schema,
+               table_name as identifier,
+               last_altered as last_modified,
+               {{ current_timestamp() }} as snapshotted_at
+        from {{ information_schema }}.tables
+        where (
+          {%- for relation in relations -%}
+            (table_schema = '{{ relation.schema }}' and
+             table_name = '{{ relation.identifier }}'){%- if not loop.last %} or {% endif -%}
+          {%- endfor -%}
+        )
+    {% endif %}
+  {%- endcall -%}
+
+  {{ return(load_result('last_modified')) }}
+
+{% endmacro %}

dbt/include/databricks/macros/adapters/relation.sql

@@ -0,0 +1,33 @@
+{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}
+    {% set tmp_identifier = base_relation.identifier ~ suffix %}
+    {%- if as_table -%}
+        {% set tmp_relation = api.Relation.create(
+            identifier=tmp_identifier,
+            schema=base_relation.schema,
+            database=base_relation.database,
+            type='table') %}
+    {%- else -%}
+        {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}
+    {%- endif -%}
+    {% do return(tmp_relation) %}
+{% endmacro %}
+
+{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}
+  {%- set target_relation = adapter.get_relation(
+            database=database,
+            schema=schema,
+            identifier=identifier,
+            needs_information=needs_information) %}
+
+  {% if target_relation %}
+    {% do return([true, target_relation]) %}
+  {% endif %}
+
+  {%- set new_relation = api.Relation.create(
+      database=database,
+      schema=schema,
+      identifier=identifier,
+      type=type
+  ) -%}
+  {% do return([false, new_relation]) %}
+{% endmacro %}

dbt/include/databricks/macros/get_custom_name/get_custom_database.sql

@@ -0,0 +1,13 @@
+{# 
+    This is identical to the implementation in dbt-core.
+    We need to override because dbt-spark overrides to something we don't like.
+#}
+
+{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}
+    {%- set default_database = target.database -%}
+    {%- if custom_database_name is none -%}
+        {{ return(default_database) }}
+    {%- else -%}
+        {{ return(custom_database_name) }}
+    {%- endif -%}
+{%- endmacro %}

dbt/include/databricks/macros/adapters.sql → dbt/include/databricks/macros/relations/constraints.sql

@@ -1,67 +1,3 @@
-{% macro databricks__file_format_clause() %}
-  {%- set file_format = config.get('file_format', default='delta') -%}
-  {%- if file_format is not none %}
-    using {{ file_format }}
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro databricks__options_clause() -%}
-  {%- set options = config.get('options') -%}
-  {%- if config.get('file_format', default='delta') == 'hudi' -%}
-    {%- set unique_key = config.get('unique_key') -%}
-    {%- if unique_key is not none and options is none -%}
-      {%- set options = {'primaryKey': config.get('unique_key')} -%}
-    {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}
-      {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}
-    {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}
-      {{ exceptions.raise_compiler_error("unique_key and options('primaryKey') should be the same column(s).") }}
-    {%- endif %}
-  {%- endif %}
-
-  {%- if options is not none %}
-    options (
-      {%- for option in options -%}
-      {{ option }} "{{ options[option] }}" {% if not loop.last %}, {% endif %}
-      {%- endfor %}
-    )
-  {%- endif %}
-{%- endmacro -%}
-
-
-{% macro tblproperties_clause() -%}
-  {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}
-{%- endmacro -%}
-
-{% macro databricks__tblproperties_clause() -%}
-  {%- set tblproperties = config.get('tblproperties') -%}
-  {%- if tblproperties is not none %}
-    tblproperties (
-      {%- for prop in tblproperties -%}
-      '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}
-      {%- endfor %}
-    )
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro liquid_clustered_cols(label, required=false) -%}
-  {{ return(adapter.dispatch('liquid_clustered_cols', 'dbt')(label, required)) }}
-{%- endmacro -%}
-
-{% macro databricks__liquid_clustered_cols(label, required=false) -%}
-  {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}
-  {%- if cols is not none %}
-    {%- if cols is string -%}
-      {%- set cols = [cols] -%}
-    {%- endif -%}
-    {{ label }} (
-    {%- for item in cols -%}
-      {{ item }}
-      {%- if not loop.last -%},{%- endif -%}
-    {%- endfor -%}
-    )
-  {%- endif %}
-{%- endmacro -%}
-
 {# Persist table-level and column-level constraints. #}
 {% macro persist_constraints(relation, model) %}
   {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}
@@ -302,123 +238,3 @@
 
   {{ return(dbt_constraints) }}
 {% endmacro %}
-
-{% macro optimize(relation) %}
-  {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}
-{% endmacro %}
-
-{%- macro databricks__optimize(relation) -%}
-  {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and 
-        var('databricks_skip_optimize', 'false')|lower != 'true' and 
-        config.get('file_format', 'delta') == 'delta' -%}
-    {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}
-      {%- call statement('run_optimize_stmt') -%}
-        {{ get_optimize_sql(relation) }}
-      {%- endcall -%}
-    {%- endif -%}
-  {%- endif -%}
-{%- endmacro -%}
-
-{%- macro get_optimize_sql(relation) %}
-  optimize {{ relation }}
-  {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
-    {%- if config.get('liquid_clustered_by', False) %}
-      {{ exceptions.warn("Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.") }}
-    {%- else %}
-      {%- set zorder = config.get('zorder', none) %}
-      {# TODO: predicates here? WHERE ...  #}
-      {%- if zorder is sequence and zorder is not string %}
-        zorder by (
-        {%- for col in zorder %}
-        {{ col }}{% if not loop.last %}, {% endif %}
-        {%- endfor %}
-        )
-      {%- else %}
-        zorder by ({{zorder}})
-      {%- endif %}
-    {%- endif %}
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro databricks__list_relations_without_caching(schema_relation) %}
-  {{ return(adapter.get_relations_without_caching(schema_relation)) }}
-{% endmacro %}
-
-{% macro show_table_extended(schema_relation) %}
-  {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}
-{% endmacro %}
-
-{% macro databricks__show_table_extended(schema_relation) %}
-  {% call statement('show_table_extended', fetch_result=True) -%}
-    show table extended in {{ schema_relation.without_identifier() }} like '{{ schema_relation.identifier }}'
-  {% endcall %}
-
-  {% do return(load_result('show_table_extended').table) %}
-{% endmacro %}
-
-{% macro show_tables(relation) %}
-  {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}
-{% endmacro %}
-
-{% macro databricks__show_tables(relation) %}
-  {% call statement('show_tables', fetch_result=True) -%}
-    show tables in {{ relation }}
-  {% endcall %}
-
-  {% do return(load_result('show_tables').table) %}
-{% endmacro %}
-
-{% macro show_views(relation) %}
-  {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}
-{% endmacro %}
-
-{% macro databricks__show_views(relation) %}
-  {% call statement('show_views', fetch_result=True) -%}
-    show views in {{ relation }}
-  {% endcall %}
-
-  {% do return(load_result('show_views').table) %}
-{% endmacro %}
-
-{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}
-    {%- set default_database = target.database -%}
-    {%- if custom_database_name is none -%}
-        {{ return(default_database) }}
-    {%- else -%}
-        {{ return(custom_database_name) }}
-    {%- endif -%}
-{%- endmacro %}
-
-{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}
-    {% set tmp_identifier = base_relation.identifier ~ suffix %}
-    {%- if as_table -%}
-        {% set tmp_relation = api.Relation.create(
-            identifier=tmp_identifier,
-            schema=base_relation.schema,
-            database=base_relation.database,
-            type='table') %}
-    {%- else -%}
-        {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}
-    {%- endif -%}
-    {% do return(tmp_relation) %}
-{% endmacro %}
-
-{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}
-  {%- set target_relation = adapter.get_relation(
-            database=database,
-            schema=schema,
-            identifier=identifier,
-            needs_information=needs_information) %}
-
-  {% if target_relation %}
-    {% do return([true, target_relation]) %}
-  {% endif %}
-
-  {%- set new_relation = api.Relation.create(
-      database=database,
-      schema=schema,
-      identifier=identifier,
-      type=type
-  ) -%}
-  {% do return([false, new_relation]) %}
-{% endmacro %}