Lean approach for process_schema_changes to simplify

dbt-labs · Nov 7, 2021 · 33c8b90 · 33c8b90
1 parent 1676882
commit 33c8b90
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 17 deletions.
diff --git a/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql
@@ -54,8 +54,7 @@
              from_relation=tmp_relation,
              to_relation=target_relation) %}
     {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}
-    {% set schema_changes_dict = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
-    {% set dest_columns = schema_changes_dict.get('source_columns') %}
+    {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
     {% if not dest_columns %}
       {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
     {% endif %}

diff --git a/core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql b/core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql
@@ -15,15 +15,15 @@
 
 {% endmacro %}
 
-{% macro compare_columns(source_columns, target_columns, should_include) %}
+{% macro diff_columns(source_columns, target_columns) %}
 
   {% set result = [] %}
   {% set source_names = source_columns | map(attribute = 'column') | list %}
   {% set target_names = target_columns | map(attribute = 'column') | list %}
 
    {# --check whether the name attribute exists in the target - this does not perform a data type check #}
    {% for sc in source_columns %}
-     {% if (sc.name in target_names) == should_include %}
+     {% if sc.name not in target_names %}
         {{ result.append(sc) }}
      {% endif %}
    {% endfor %}
@@ -32,14 +32,6 @@
 
 {% endmacro %}
 
-{% macro diff_columns(source_columns, target_columns) %}
-  {{ return(compare_columns(source_columns, target_columns, false) ) }}
-{% endmacro %}
-
-{% macro intersect_columns(source_columns, target_columns) %}
-  {{ return(compare_columns(source_columns, target_columns, true) ) }}
-{% endmacro %}
-
 {% macro diff_column_data_types(source_columns, target_columns) %}
 
   {% set result = [] %}
@@ -65,8 +57,7 @@
   {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
   {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
   {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
-  {%- set in_target_and_source = intersect_columns(target_columns, source_columns) -%}
-
+
   {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}
 
   {% if source_not_in_target != [] %}
@@ -81,7 +72,6 @@
     'schema_changed': schema_changed,
     'source_not_in_target': source_not_in_target,
     'target_not_in_source': target_not_in_source,
-    'in_target_and_source': in_target_and_source,
     'source_columns': source_columns,
     'target_columns': target_columns,
     'new_target_types': new_target_types
@@ -165,7 +155,7 @@
           {% endset %}
 
           {% do exceptions.raise_compiler_error(fail_msg) %}
-        
+
         {# -- unless we ignore, run the sync operation per the config #}
         {% else %}
 
@@ -175,7 +165,7 @@
 
       {% endif %}
 
-      {{ return(schema_changes_dict) }}
+      {{ return(schema_changes_dict['source_columns']) }}
 
     {% endif %}