Skip to content

Commit

Permalink
Add metadata to tests, add a native env
Browse files Browse the repository at this point in the history
Render test arguments in the native env and pass them along to the context
added/fixed tests
Update changelog
  • Loading branch information
Jacob Beck committed Mar 24, 2020
1 parent 31025a0 commit dc65118
Show file tree
Hide file tree
Showing 18 changed files with 296 additions and 50 deletions.
6 changes: 3 additions & 3 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
## dbt next (release TBD)
### Features
- Added --fail-fast argument for dbt run and dbt test to fail on first test failure or runtime error. ([#1649](https://github.com/fishtown-analytics/dbt/issues/1649))
- Support for appending query comments to SQL queries. ([#2138](https://github.com/fishtown-analytics/dbt/issues/2138))
- Added --fail-fast argument for dbt run and dbt test to fail on first test failure or runtime error. ([#1649](https://github.com/fishtown-analytics/dbt/issues/1649), [#2224](https://github.com/fishtown-analytics/dbt/pull/2224))
- Support for appending query comments to SQL queries. ([#2138](https://github.com/fishtown-analytics/dbt/issues/2138), [#2199](https://github.com/fishtown-analytics/dbt/pull/2199))
- Added a `get-manifest` API call. ([#2168](https://github.com/fishtown-analytics/dbt/issues/2168), [#2232](https://github.com/fishtown-analytics/dbt/pull/2232))
- Users can now use jinja as arguments to tests. Test arguments are rendered in the native context and injected into the test execution context directly. ([#2149](https://github.com/fishtown-analytics/dbt/issues/2149), [#2220](https://github.com/fishtown-analytics/dbt/pull/2220))

### Fixes
- When a jinja value is undefined, give a helpful error instead of failing with cryptic "cannot pickle ParserMacroCapture" errors ([#2110](https://github.com/fishtown-analytics/dbt/issues/2110), [#2184](https://github.com/fishtown-analytics/dbt/pull/2184))
- Added timeout to registry download call ([#2195](https://github.com/fishtown-analytics/dbt/issues/2195), [#2228](https://github.com/fishtown-analytics/dbt/pull/2228))

Contributors:
- [@raalsky](https://github.com/Raalsky) ([#2224](https://github.com/fishtown-analytics/dbt/pull/2224), [#2228](https://github.com/fishtown-analytics/dbt/pull/2228))
- [@raalsky](https://github.com/Raalsky) ([#2224](https://github.com/fishtown-analytics/dbt/pull/2224))
- [@ilkinulas](https://github.com/ilkinulas) [#2199](https://github.com/fishtown-analytics/dbt/pull/2199)


Expand Down
95 changes: 87 additions & 8 deletions core/dbt/clients/jinja.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import codecs
import linecache
import os
import re
import tempfile
import threading
from contextlib import contextmanager
Expand All @@ -10,15 +11,19 @@

import jinja2
import jinja2.ext
import jinja2.nativetypes # type: ignore
import jinja2.nodes
import jinja2.parser
import jinja2.sandbox

from dbt.utils import (
get_dbt_macro_name, get_docs_macro_name, get_materialization_macro_name
get_dbt_macro_name, get_docs_macro_name, get_materialization_macro_name,
deep_map
)

from dbt.clients._jinja_blocks import BlockIterator, BlockData, BlockTag
from dbt.contracts.graph.compiled import CompiledSchemaTestNode
from dbt.contracts.graph.parsed import ParsedSchemaTestNode
from dbt.exceptions import (
InternalException, raise_compiler_error, CompilationException,
invalid_materialization_argument, MacroReturn
Expand Down Expand Up @@ -93,6 +98,17 @@ def _compile(self, source, filename):
return super()._compile(source, filename) # type: ignore


class NativeSandboxEnvironment(MacroFuzzEnvironment):
code_generator_class = jinja2.nativetypes.NativeCodeGenerator


class NativeSandboxTemplate(jinja2.nativetypes.NativeTemplate): # mypy: ignore
environment_class = NativeSandboxEnvironment


NativeSandboxEnvironment.template_class = NativeSandboxTemplate # type: ignore


class TemplateCache:

def __init__(self):
Expand Down Expand Up @@ -348,7 +364,11 @@ def __reduce__(self):
return Undefined


def get_environment(node=None, capture_macros=False):
def get_environment(
node=None,
capture_macros: bool = False,
native: bool = False,
) -> jinja2.Environment:
args: Dict[str, List[Union[str, Type[jinja2.ext.Extension]]]] = {
'extensions': ['jinja2.ext.do']
}
Expand All @@ -359,7 +379,13 @@ def get_environment(node=None, capture_macros=False):
args['extensions'].append(MaterializationExtension)
args['extensions'].append(DocumentationExtension)

return MacroFuzzEnvironment(**args)
env_cls: Type[jinja2.Environment]
if native:
env_cls = NativeSandboxEnvironment
else:
env_cls = MacroFuzzEnvironment

return env_cls(**args)


@contextmanager
Expand All @@ -378,21 +404,39 @@ def parse(string):
return get_environment().parse(str(string))


def get_template(string, ctx, node=None, capture_macros=False):
def get_template(
string: str,
ctx: Dict[str, Any],
node=None,
capture_macros: bool = False,
native: bool = False,
):
with catch_jinja(node):
env = get_environment(node, capture_macros)
env = get_environment(node, capture_macros, native=native)

template_source = str(string)
return env.from_string(template_source, globals=ctx)


def render_template(template, ctx, node=None):
def render_template(template, ctx: Dict[str, Any], node=None) -> str:
with catch_jinja(node):
return template.render(ctx)


def get_rendered(string, ctx, node=None, capture_macros=False):
template = get_template(string, ctx, node, capture_macros=capture_macros)
def get_rendered(
string: str,
ctx: Dict[str, Any],
node=None,
capture_macros: bool = False,
native: bool = False,
) -> str:
template = get_template(
string,
ctx,
node,
capture_macros=capture_macros,
native=native,
)

return render_template(template, ctx, node)

Expand Down Expand Up @@ -424,3 +468,38 @@ def extract_toplevel_blocks(
allowed_blocks=allowed_blocks,
collect_raw_data=collect_raw_data
)


SCHEMA_TEST_KWARGS_NAME = '_dbt_schema_test_kwargs'


def add_rendered_test_kwargs(
context: Dict[str, Any],
node: Union[ParsedSchemaTestNode, CompiledSchemaTestNode],
capture_macros: bool = False,
) -> None:
"""Render each of the test kwargs in the given context using the native
renderer, then insert that value into the given context as the special test
keyword arguments member.
"""
looks_like_func = r'^\s*(env_var|ref|var|source|doc)\s*\(.+\)\s*$'

# we never care about the keypath
def _convert_function(value: Any, _: Any) -> Any:
if isinstance(value, str):
if re.match(looks_like_func, value) is not None:
# curly braces to make rendering happy
value = f'{{{{ {value} }}}}'
new_value = get_rendered(
value, context, node, capture_macros=capture_macros,
native=True
)
# this is an ugly way of checking if the value was a quoted str
# (we don't want to unquote column names!)
if not (len(value) >= 2 and new_value == value[1:-1]):
value = new_value

return value

kwargs = deep_map(_convert_function, node.test_metadata.kwargs)
context[SCHEMA_TEST_KWARGS_NAME] = kwargs
31 changes: 26 additions & 5 deletions core/dbt/compilation.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import itertools
import os
from collections import defaultdict
from typing import List, Dict
from typing import List, Dict, Any

import dbt.utils
import dbt.include
Expand All @@ -11,11 +11,17 @@
from dbt.linker import Linker

from dbt.context.providers import generate_runtime_model
from dbt.contracts.graph.manifest import Manifest
import dbt.contracts.project
import dbt.exceptions
import dbt.flags
import dbt.config
from dbt.contracts.graph.compiled import InjectedCTE, COMPILED_TYPES
from dbt.contracts.graph.compiled import (
InjectedCTE,
COMPILED_TYPES,
NonSourceCompiledNode,
CompiledSchemaTestNode,
)
from dbt.contracts.graph.parsed import ParsedNode

from dbt.logger import GLOBAL_LOGGER as logger
Expand Down Expand Up @@ -130,6 +136,22 @@ def initialize(self):
dbt.clients.system.make_directory(self.config.target_path)
dbt.clients.system.make_directory(self.config.modules_path)

def _create_node_context(
self,
node: NonSourceCompiledNode,
manifest: Manifest,
extra_context: Dict[str, Any],
) -> Dict[str, Any]:
context = generate_runtime_model(
node, self.config, manifest
)
context.update(extra_context)
if isinstance(node, CompiledSchemaTestNode):
# for test nodes, add a special keyword args value to the context
dbt.clients.jinja.add_rendered_test_kwargs(context, node)

return context

def compile_node(self, node, manifest, extra_context=None):
if extra_context is None:
extra_context = {}
Expand All @@ -146,10 +168,9 @@ def compile_node(self, node, manifest, extra_context=None):
})
compiled_node = _compiled_type_for(node).from_dict(data)

context = generate_runtime_model(
compiled_node, self.config, manifest
context = self._create_node_context(
compiled_node, manifest, extra_context
)
context.update(extra_context)

compiled_node.compiled_sql = dbt.clients.jinja.get_rendered(
node.raw_sql,
Expand Down
4 changes: 2 additions & 2 deletions core/dbt/config/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ def _preprocess(project_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Pre-process certain special keys to convert them from None values
into empty containers, and to turn strings into arrays of strings.
"""
handlers: Dict[Tuple[str, ...], Callable[[Any], Any]] = {
handlers: Dict[Tuple[Union[str, int], ...], Callable[[Any], Any]] = {
('on-run-start',): _list_if_none_or_string,
('on-run-end',): _list_if_none_or_string,
}
Expand All @@ -286,7 +286,7 @@ def _preprocess(project_dict: Dict[str, Any]) -> Dict[str, Any]:
handlers[(k, 'post-hook')] = _list_if_none_or_string
handlers[('seeds', 'column_types')] = _dict_if_none

def converter(value: Any, keypath: Tuple[str, ...]) -> Any:
def converter(value: Any, keypath: Tuple[Union[str, int], ...]) -> Any:
if keypath in handlers:
handler = handlers[keypath]
return handler(value)
Expand Down
14 changes: 11 additions & 3 deletions core/dbt/contracts/graph/compiled.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@ class CompiledSnapshotNode(CompiledNode):
@dataclass
class CompiledDataTestNode(CompiledNode):
resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
column_name: Optional[str] = None
config: TestConfig = field(default_factory=TestConfig)


Expand Down Expand Up @@ -237,8 +236,7 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
return cls.from_dict(compiled.to_dict(), validate=False)


# This is anything that can be in manifest.nodes and isn't a Source.
NonSourceNode = Union[
NonSourceCompiledNode = Union[
CompiledAnalysisNode,
CompiledDataTestNode,
CompiledModelNode,
Expand All @@ -247,6 +245,9 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
CompiledSchemaTestNode,
CompiledSeedNode,
CompiledSnapshotNode,
]

NonSourceParsedNode = Union[
ParsedAnalysisNode,
ParsedDataTestNode,
ParsedModelNode,
Expand All @@ -257,6 +258,13 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
ParsedSnapshotNode,
]


# This is anything that can be in manifest.nodes and isn't a Source.
NonSourceNode = Union[
NonSourceCompiledNode,
NonSourceParsedNode,
]

# We allow either parsed or compiled nodes, or parsed sources, as some
# 'compile()' calls in the runner actually just return the original parsed
# node they were given.
Expand Down
1 change: 0 additions & 1 deletion core/dbt/contracts/graph/parsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,6 @@ class HasTestMetadata(JsonSchemaMixin):
@dataclass
class ParsedDataTestNode(ParsedNode):
resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
column_name: Optional[str] = None
config: TestConfig = field(default_factory=TestConfig)


Expand Down
7 changes: 3 additions & 4 deletions core/dbt/node_runners.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,10 @@
)
from dbt.exceptions import (
NotImplementedException, CompilationException, RuntimeException,
InternalException, missing_materialization
InternalException, missing_materialization, raise_compiler_error
)
from dbt.logger import GLOBAL_LOGGER as logger
from dbt.node_types import NodeType
import dbt.exceptions
import dbt.tracking
import dbt.ui.printer
import dbt.flags
Expand Down Expand Up @@ -570,7 +569,7 @@ def execute_data_test(self, test: CompiledDataTestNode):
num_cols = len(table.columns)
# since we just wrapped our query in `select count(*)`, we are in
# big trouble!
raise dbt.exceptions.InternalException(
raise InternalException(
f"dbt itnernally failed to execute {test.unique_id}: "
f"Returned {num_rows} rows and {num_cols} cols, but expected "
f"1 row and 1 column"
Expand All @@ -587,7 +586,7 @@ def execute_schema_test(self, test: CompiledSchemaTestNode):
num_rows = len(table.rows)
if num_rows != 1:
num_cols = len(table.columns)
dbt.exceptions.raise_compiler_error(
raise_compiler_error(
f"Bad test {test.test_metadata.name}: "
f"Returned {num_rows} rows and {num_cols} cols, but expected "
f"1 row and 1 column"
Expand Down
Loading

0 comments on commit dc65118

Please sign in to comment.