dbt test --store-failures

core/dbt/compilation.py

@@ -182,8 +182,7 @@ def add_ephemeral_prefix(self, name: str):
 
     def _get_relation_name(self, node: ParsedNode):
         relation_name = None
-        if (node.resource_type in NodeType.refable() and
-                not node.is_ephemeral_model):
+        if node.is_relational and not node.is_ephemeral_model:
             adapter = get_adapter(self.config)
             relation_cls = adapter.Relation
             relation_name = str(relation_cls.create_from(self.config, node))

core/dbt/contracts/graph/model_config.py

@@ -436,6 +436,7 @@ class SeedConfig(NodeConfig):
 
 @dataclass
 class TestConfig(NodeConfig):
+    schema: str = 'dbt_test__audit'
     materialized: str = 'test'
     severity: Severity = Severity('ERROR')
 

core/dbt/contracts/graph/parsed.py

@@ -115,6 +115,10 @@ class ParsedNodeMixins(dbtClassMixin):
     def is_refable(self):
         return self.resource_type in NodeType.refable()
 
+    @property
+    def is_relational(self):
+        return self.resource_type in NodeType.relational()
+
     @property
     def is_ephemeral(self):
         return self.config.materialized == 'ephemeral'

core/dbt/flags.py

@@ -16,6 +16,7 @@
 WRITE_JSON = None
 PARTIAL_PARSE = None
 USE_COLORS = None
+STORE_FAILURES = None
 
 
 def env_set_truthy(key: str) -> Optional[str]:
@@ -53,7 +54,7 @@ def _get_context():
 
 def reset():
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
-        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS
+        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, STORE_FAILURES
 
     STRICT_MODE = False
     FULL_REFRESH = False
@@ -64,11 +65,12 @@ def reset():
     PARTIAL_PARSE = False
     MP_CONTEXT = _get_context()
     USE_COLORS = True
+    STORE_FAILURES = False
 
 
 def set_from_args(args):
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
-        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS
+        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, STORE_FAILURES
 
     USE_CACHE = getattr(args, 'use_cache', USE_CACHE)
 
@@ -91,6 +93,8 @@ def set_from_args(args):
     if use_colors_override is not None:
         USE_COLORS = use_colors_override
 
+    STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
+
 
 # initialize everything to the defaults on module load
 reset()

core/dbt/include/global_project/macros/materializations/test.sql

@@ -1,10 +1,46 @@
 {%- materialization test, default -%}
 
+  {% set relations = [] %}
+
+  {% if flags.STORE_FAILURES %}
+
+    {% set identifier = model['name'] %}
+    {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}
+    {% set target_relation = api.Relation.create(
+        identifier=identifier, schema=schema, database=database, type='table') -%} %}
+
+    {% if old_relation %}
+        {% do adapter.drop_relation(old_relation) %}
+    {% endif %}
+
+    {% call statement(auto_begin=True) %}
+        {{ create_table_as(False, target_relation, sql) }}
+    {% endcall %}
+
+    {% do relations.append(target_relation) %}
+
+    {% set main_sql %}
+        select count(*) as validation_errors
+        from {{ target_relation }}
+    {% endset %}
+
+    {{ adapter.commit() }}
+
+  {% else %}
+
+      {% set main_sql %}
+          select count(*) as validation_errors
+          from (
+            {{ sql }}
+          ) _dbt_internal_test
+      {% endset %}
+
+  {% endif %}
+
   {% call statement('main', fetch_result=True) -%}
-    select count(*) as validation_errors
-    from (
-      {{ sql }}
-    ) _dbt_internal_test
+    {{ main_sql }}
   {%- endcall %}
+
+  {{ return({'relations': relations}) }}
 
 {%- endmaterialization -%}

core/dbt/include/global_project/macros/schema_tests/accepted_values.sql

@@ -6,17 +6,20 @@
 
 with all_values as (
 
-    select distinct
-        {{ column_name }} as value_field
+    select
+        {{ column_name }} as value_field,
+        count(*) as num
 
     from {{ model }}
+    group by 1
 
 ),
 
 validation_errors as (
 
     select
-        value_field
+        value_field,
+        num
 
     from all_values
     where value_field not in (

core/dbt/include/global_project/macros/schema_tests/unique.sql

@@ -7,7 +7,8 @@ select *
 from (
 
     select
-        {{ column_name }}
+        {{ column_name }} as value,
+        count(*) as num
 
     from {{ model }}
     where {{ column_name }} is not null

core/dbt/main.py

@@ -719,6 +719,13 @@ def _build_test_subparser(subparsers, base_subparser):
         Stop execution upon a first test failure.
         '''
     )
+    sub.add_argument(
+        '--store-failures',
+        action='store_true',
+        help='''
+        Store test results (failing rows) in the database
+        '''
+    )
 
     sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
     return sub

core/dbt/node_types.py

@@ -1,6 +1,7 @@
 from typing import List
 
 from dbt.dataclass_schema import StrEnum
+from dbt import flags
 
 
 class NodeType(StrEnum):
@@ -49,6 +50,10 @@ def documentable(cls) -> List['NodeType']:
             cls.Exposure
         ]
 
+    @classmethod
+    def relational(cls) -> List['NodeType']:
+        return cls.refable() + ([cls.Test] if flags.STORE_FAILURES else [])
+
     def pluralize(self) -> str:
         if self == 'analysis':
             return 'analyses'

core/dbt/task/printer.py

@@ -10,6 +10,7 @@
 from dbt.tracking import InvocationProcessor
 from dbt import ui
 from dbt import utils
+from dbt import flags
 
 from dbt.contracts.results import (
     FreshnessStatus, NodeResult, NodeStatus, TestStatus
@@ -306,6 +307,13 @@ def print_run_result_error(
             logger.info("  compiled SQL at {}".format(
                 result.node.build_path))
 
+        if flags.STORE_FAILURES:
+            with TextOnly():
+                logger.info("")
+            msg = f"select * from {result.node.relation_name}"
+            border = '-' * len(msg)
+            logger.info(f"  See test failures:\n  {border}\n  {msg}\n  {border}")
+
     elif result.message is not None:
         first = True
         for line in result.message.split("\n"):

core/dbt/task/runnable.py

@@ -455,7 +455,7 @@ def get_model_schemas(
         for node in self.manifest.nodes.values():
             if node.unique_id not in selected_uids:
                 continue
-            if node.is_refable and not node.is_ephemeral:
+            if node.is_relational and not node.is_ephemeral:
                 relation = adapter.Relation.create_from(self.config, node)
                 result.add(relation.without_identifier())
 
@@ -524,8 +524,6 @@ def create_schema(relation: BaseRelation) -> None:
 
                 db_schema = (db_lower, schema.lower())
                 if db_schema not in existing_schemas_lowered:
-                    existing_schemas_lowered.add(db_schema)
-
                     fut = tpe.submit_connected(
                         adapter, f'create_{info.database or ""}_{info.schema}',
                         create_schema, info

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -999,7 +999,7 @@ def rendered_tst_config(self, **updates):
             'severity': 'ERROR',
             'full_refresh': None,
             'database': None,
-            'schema': None,
+            'schema': 'dbt_test__audit',
             'alias': None,
         }
         result.update(updates)
@@ -1065,6 +1065,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
         snapshot_path = self.dir(os.path.join('snapshot', 'snapshot_seed.sql'))
 
         my_schema_name = self.unique_schema()
+        test_audit_schema = my_schema_name + '_dbt_test__audit'
 
         if model_database is None:
             model_database = self.alternative_database
@@ -1357,7 +1358,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
                     'relation_name': None,
                     'resource_type': 'test',
                     'root_path': self.test_root_realpath,
-                    'schema': my_schema_name,
+                    'schema': test_audit_schema,
                     'database': self.default_database,
                     'tags': ['schema'],
                     'meta': {},
@@ -1444,7 +1445,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
                     'relation_name': None,
                     'resource_type': 'test',
                     'root_path': self.test_root_realpath,
-                    'schema': my_schema_name,
+                    'schema': test_audit_schema,
                     'database': self.default_database,
                     'tags': ['schema'],
                     'meta': {},
@@ -1488,7 +1489,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
                     'relation_name': None,
                     'resource_type': 'test',
                     'root_path': self.test_root_realpath,
-                    'schema': my_schema_name,
+                    'schema': test_audit_schema,
                     'database': self.default_database,
                     'tags': ['schema'],
                     'meta': {},

test/integration/047_dbt_ls_test/test_ls.py

@@ -344,7 +344,7 @@ def expect_test_output(self):
                         'persist_docs': {},
                         'full_refresh': None,
                         'database': None,
-                        'schema': None,
+                        'schema': 'dbt_test__audit',
                         'alias': None,
                     },
                     'alias': 'not_null_outer_id',
@@ -368,7 +368,7 @@ def expect_test_output(self):
                         'persist_docs': {},
                         'full_refresh': None,
                         'database': None,
-                        'schema': None,
+                        'schema': 'dbt_test__audit',
                         'alias': None,
                     },
                     'alias': 't',
@@ -392,7 +392,7 @@ def expect_test_output(self):
                         'persist_docs': {},
                         'full_refresh': None,
                         'database': None,
-                        'schema': None,
+                        'schema': 'dbt_test__audit',
                         'alias': None,
                     },
                     'alias': 'unique_outer_id',

test/integration/066_test_selection_tests/test_selection_expansion.py

@@ -7,7 +7,7 @@
 class TestSelectionExpansion(DBTIntegrationTest):
     @property
     def schema(self):
-        return "test_selection_expansion_065"
+        return "test_selection_expansion_066"
 
     @property
     def models(self):

test/integration/067_store_test_failures_tests/data/expected/expected_failing_test.csv

@@ -0,0 +1,11 @@
+id,first_name,last_name,email,gender,ip_address
+1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168
+2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35
+3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
+4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175
+5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136
+6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220
+7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64
+8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13
+9,Gary,Day,gday8@nih.gov,Male,35.81.68.186
+10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100

test/integration/067_store_test_failures_tests/data/expected/expected_not_null_problematic_model_id.csv

@@ -0,0 +1,3 @@
+id,first_name,last_name,email,gender,ip_address
+,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
+,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175

test/integration/067_store_test_failures_tests/data/expected/expected_unique_problematic_model_id.csv

@@ -0,0 +1,3 @@
+value,num
+2,2
+1,2

test/integration/067_store_test_failures_tests/data/people.csv

@@ -0,0 +1,11 @@
+id,first_name,last_name,email,gender,ip_address
+1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168
+2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35
+3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
+4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175
+5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136
+6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220
+7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64
+8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13
+9,Gary,Day,gday8@nih.gov,Male,35.81.68.186
+10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100

test/integration/067_store_test_failures_tests/models/fine_model.sql

@@ -0,0 +1 @@
+select * from {{ ref('people') }}

test/integration/067_store_test_failures_tests/models/problematic_model.sql

@@ -0,0 +1,11 @@
+select * from {{ ref('people') }}
+
+union all
+
+select * from {{ ref('people') }}
+where id in (1,2)
+
+union all
+
+select null as id, first_name, last_name, email, gender, ip_address from {{ ref('people') }}
+where id in (3,4)

test/integration/067_store_test_failures_tests/models/schema.yml

@@ -0,0 +1,17 @@
+version: 2
+
+models:
+
+  - name: fine_model
+    columns:
+      - name: id
+        tests:
+          - unique
+          - not_null
+
+  - name: problematic_model
+    columns:
+      - name: id
+        tests:
+          - unique
+          - not_null