From b5f336b39a01f83e9127c09f19e5617ef59d4acb Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Mon, 8 Nov 2021 11:34:45 +0100 Subject: [PATCH] Add tests on incremental materialization (#37) --- ...remental_append_new_columns_remove_one.sql | 28 +++++++++++++++++++ ...l_append_new_columns_remove_one_target.sql | 19 +++++++++++++ .../models/schema.yml | 16 ++++++++++- .../test_incremental_schema.py | 16 +++++++++++ ...remental_append_new_columns_remove_one.sql | 1 + ...l_append_new_columns_remove_one_target.sql | 1 + 6 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql create mode 100644 tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql create mode 100644 tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one.sql create mode 100644 tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one_target.sql diff --git a/tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql b/tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql new file mode 100644 index 000000000..19c8ea616 --- /dev/null +++ b/tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +{% set string_type = 'string' if target.type == 'bigquery' else 'varchar(10)' %} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field3 as {{string_type}}) as field3, + cast(field4 as {{string_type}}) as field4 +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, + cast(field1 as {{string_type}}) as field1, + cast(field2 as {{string_type}}) as field2 +FROM source_data where id <= 3 + +{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql b/tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql new file mode 100644 index 000000000..419fdf96b --- /dev/null +++ b/tests/integration/incremental_schema_tests/models/incremental_append_new_columns_remove_one_target.sql @@ -0,0 +1,19 @@ +{{ + config(materialized='table') +}} + +{% set string_type = 'string' if target.type == 'bigquery' else 'varchar(10)' %} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +select id, + cast(field1 as {{string_type}}) as field1, + cast(CASE WHEN id > 3 THEN NULL ELSE field2 END as {{string_type}}) AS field2, + cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3, + cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 + +from source_data \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/models/schema.yml b/tests/integration/incremental_schema_tests/models/schema.yml index 5546314e4..6d2a85bea 100644 --- a/tests/integration/incremental_schema_tests/models/schema.yml +++ b/tests/integration/incremental_schema_tests/models/schema.yml @@ -35,7 +35,21 @@ models: tags: [column_level_tag] tests: - unique - + + - name: incremental_append_new_columns_remove_one + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + + - name: incremental_append_new_columns_remove_one_target + columns: + - name: id + tags: [column_level_tag] + tests: + - unique + - name: incremental_sync_all_columns columns: - name: id diff --git a/tests/integration/incremental_schema_tests/test_incremental_schema.py b/tests/integration/incremental_schema_tests/test_incremental_schema.py index 5d803c48f..7dff20c1e 100644 --- a/tests/integration/incremental_schema_tests/test_incremental_schema.py +++ b/tests/integration/incremental_schema_tests/test_incremental_schema.py @@ -91,6 +91,21 @@ def run_incremental_append_new_columns(self): self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected, compare_source, compare_target) + def run_incremental_append_new_columns_remove_one(self): + select = 'model_a incremental_append_new_columns_remove_one incremental_append_new_columns_remove_one_target' + compare_source = 'incremental_append_new_columns_remove_one' + compare_target = 'incremental_append_new_columns_remove_one_target' + exclude = None + expected = [ + 'select_from_a', + 'select_from_incremental_append_new_columns_remove_one', + 'select_from_incremental_append_new_columns_remove_one_target', + 'unique_model_a_id', + 'unique_incremental_append_new_columns_remove_one_id', + 'unique_incremental_append_new_columns_remove_one_target_id' + ] + self.run_tests_and_assert(select, exclude, expected, compare_source, compare_target) + def run_incremental_sync_all_columns(self): select = 'model_a incremental_sync_all_columns incremental_sync_all_columns_target' compare_source = 'incremental_sync_all_columns' @@ -121,6 +136,7 @@ def test__redshift__run_incremental_ignore(self): @use_profile('redshift') def test__redshift__run_incremental_append_new_columns(self): self.run_incremental_append_new_columns() + self.run_incremental_append_new_columns_remove_one() @use_profile('redshift') def test__redshift__run_incremental_sync_all_columns(self): diff --git a/tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one.sql b/tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one.sql new file mode 100644 index 000000000..06d52c6d6 --- /dev/null +++ b/tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_append_new_columns_remove_one') }} where false \ No newline at end of file diff --git a/tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one_target.sql b/tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one_target.sql new file mode 100644 index 000000000..07d2412b0 --- /dev/null +++ b/tests/integration/incremental_schema_tests/tests/select_from_incremental_append_new_columns_remove_one_target.sql @@ -0,0 +1 @@ +select * from {{ ref('incremental_append_new_columns_remove_one_target') }} where false \ No newline at end of file