Use common columns for incremental schema changes

dbt-labs · Oct 31, 2021 · 5769d61 · 5769d61
1 parent dd7af47
commit 5769d61
Showing 1 changed file with 21 additions and 4 deletions.
diff --git a/core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql b/core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql
@@ -15,15 +15,15 @@
 
 {% endmacro %}
 
-{% macro diff_columns(source_columns, target_columns) %}
+{% macro compare_columns(source_columns, target_columns, should_include) %}
 
   {% set result = [] %}
   {% set source_names = source_columns | map(attribute = 'column') | list %}
   {% set target_names = target_columns | map(attribute = 'column') | list %}
 
    {# --check whether the name attribute exists in the target - this does not perform a data type check #}
    {% for sc in source_columns %}
-     {% if sc.name not in target_names %}
+     {% if (sc.name in target_names) == should_include %}
         {{ result.append(sc) }}
      {% endif %}
    {% endfor %}
@@ -32,6 +32,14 @@
 
 {% endmacro %}
 
+{% macro diff_columns(source_columns, target_columns) %}
+  {{ return(compare_columns(source_columns, target_columns, false) ) }}
+{% endmacro %}
+
+{% macro intersect_columns(source_columns, target_columns) %}
+  {{ return(compare_columns(source_columns, target_columns, true) ) }}
+{% endmacro %}
+
 {% macro diff_column_data_types(source_columns, target_columns) %}
 
   {% set result = [] %}
@@ -57,7 +65,8 @@
   {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
   {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
   {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
-
+  {%- set in_target_and_source = intersect_columns(target_columns, source_columns) -%}
+
   {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}
 
   {% if source_not_in_target != [] %}
@@ -72,6 +81,8 @@
     'schema_changed': schema_changed,
     'source_not_in_target': source_not_in_target,
     'target_not_in_source': target_not_in_source,
+    'in_target_and_source': in_target_and_source,
+    'target_columns': target_columns,
     'new_target_types': new_target_types
   } %}
 
@@ -132,7 +143,11 @@
 
 {% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
 
-    {% if on_schema_change != 'ignore' %}
+    {% if on_schema_change = 'ignore' %}
+
+     {{ return({}) }}
+
+    {% else %}
 
       {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
 
@@ -158,6 +173,8 @@
         {% endif %}
 
       {% endif %}
+
+      {{ return(schema_changes_dict) }}
 
     {% endif %}