diff --git a/.changes/unreleased/Under the Hood-20220606-230353.yaml b/.changes/unreleased/Under the Hood-20220606-230353.yaml new file mode 100644 index 00000000000..16c58f6670a --- /dev/null +++ b/.changes/unreleased/Under the Hood-20220606-230353.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Update context readme + clean up context code" +time: 2022-06-06T23:03:53.022568+02:00 +custom: + Author: jtcohen6 + Issue: "4796" + PR: "5334" diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index e6c2d4523a0..dbfecb905e5 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -1,12 +1,15 @@ from typing import Dict, Any, Tuple, Optional, Union, Callable +import re +import os from dbt.clients.jinja import get_rendered, catch_jinja from dbt.context.target import TargetContext -from dbt.context.secret import SecretContext +from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER from dbt.context.base import BaseContext from dbt.contracts.connection import HasCredentials from dbt.exceptions import DbtProjectError, CompilationException, RecursionException from dbt.utils import deep_map_render +from dbt.logger import SECRET_ENV_PREFIX Keypath = Tuple[Union[str, int], ...] @@ -174,6 +177,23 @@ def __init__(self, cli_vars: Dict[str, Any] = {}) -> None: def name(self): return "Secret" + def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any: + rendered = super().render_value(value, keypath) + if SECRET_ENV_PREFIX in str(rendered): + search_group = f"({SECRET_ENV_PREFIX}(.*))" + pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$") + m = re.search( + pattern, + rendered, + ) + if m: + found = m.group(1) + value = os.environ[found] + replace_this = SECRET_PLACEHOLDER.format(found) + return rendered.replace(replace_this, value) + else: + return rendered + class ProfileRenderer(SecretRenderer): @property diff --git a/core/dbt/context/README.md b/core/dbt/context/README.md index 37eedae7a87..03762bc9785 100644 --- a/core/dbt/context/README.md +++ b/core/dbt/context/README.md @@ -1 +1,51 @@ # Contexts and Jinja rendering + +Contexts are used for Jinja rendering. They include context methods, executable macros, and various settings that are available in Jinja. + +The most common entrypoint to Jinja rendering in dbt is a method named `get_rendered`, which takes two arguments: templated code (string), and a context used to render it (dictionary). + +The context is the bundle of information that is in "scope" when rendering Jinja-templated code. For instance, imagine a simple Jinja template: +``` +{% set new_value = some_macro(some_variable) %} +``` +Both `some_macro()` and `some_variable` must be defined in that context. Otherwise, it will raise an error when rendering. + +Different contexts are used in different places because we allow access to different methods and data in different places. Executable SQL, for example, includes all available macros and the model being run. The variables and macros in scope for Jinja defined in yaml files is much more limited. + +### Implementation + +The context that is passed to Jinja is always in a dictionary format, not an actual class, so a `to_dict()` is executed on a context class before it is used for rendering. + +Each context has a `generate__context` function to create the context. `ProviderContext` subclasses have different generate functions for parsing and for execution, so that certain functions (notably `ref`, `source`, and `config`) can return different results + +### Hierarchy + +All contexts inherit from the `BaseContext`, which includes "pure" methods (e.g. `tojson`), `env_var()`, and `var()` (but only CLI values, passed via `--vars`). + +Methods available in parent contexts are also available in child contexts. + +``` + BaseContext -- core/dbt/context/base.py + SecretContext -- core/dbt/context/secret.py + TargetContext -- core/dbt/context/target.py + ConfiguredContext -- core/dbt/context/configured.py + SchemaYamlContext -- core/dbt/context/configured.py + DocsRuntimeContext -- core/dbt/context/configured.py + MacroResolvingContext -- core/dbt/context/configured.py + ManifestContext -- core/dbt/context/manifest.py + QueryHeaderContext -- core/dbt/context/manifest.py + ProviderContext -- core/dbt/context/provider.py + MacroContext -- core/dbt/context/provider.py + ModelContext -- core/dbt/context/provider.py + TestContext -- core/dbt/context/provider.py +``` + +### Contexts for configuration + +Contexts for rendering "special" `.yml` (configuration) files: +- `SecretContext`: Supports "secret" env vars, which are prefixed with `DBT_ENV_SECRET_`. Used for rendering in `profiles.yml` and `packages.yml` ONLY. Secrets defined elsewhere will raise explicit errors. +- `TargetContext`: The same as `Base`, plus `target` (connection profile). Used most notably in `dbt_project.yml` and `selectors.yml`. + +Contexts for other `.yml` files in the project: +- `SchemaYamlContext`: Supports `vars` declared on the CLI and in `dbt_project.yml`. Does not support custom macros, beyond `var()` + `env_var()` methods. Used for all `.yml` files, to define properties and configuration. +- `DocsRuntimeContext`: Standard `.yml` file context, plus `doc()` method (with all `docs` blocks in scope). Used to resolve `description` properties. diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 355937bb4fd..b64030cc86e 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -24,38 +24,7 @@ import datetime import re -# Contexts in dbt Core -# Contexts are used for Jinja rendering. They include context methods, -# executable macros, and various settings that are available in Jinja. -# -# Different contexts are used in different places because we allow access -# to different methods and data in different places. Executable SQL, for -# example, includes the available macros and the model, while Jinja in -# yaml files is more limited. -# -# The context that is passed to Jinja is always in a dictionary format, -# not an actual class, so a 'to_dict()' is executed on a context class -# before it is used for rendering. -# -# Each context has a generate__context function to create the context. -# ProviderContext subclasses have different generate functions for -# parsing and for execution. -# -# Context class hierarchy -# -# BaseContext -- core/dbt/context/base.py -# SecretContext -- core/dbt/context/secret.py -# TargetContext -- core/dbt/context/target.py -# ConfiguredContext -- core/dbt/context/configured.py -# SchemaYamlContext -- core/dbt/context/configured.py -# DocsRuntimeContext -- core/dbt/context/configured.py -# MacroResolvingContext -- core/dbt/context/configured.py -# ManifestContext -- core/dbt/context/manifest.py -# QueryHeaderContext -- core/dbt/context/manifest.py -# ProviderContext -- core/dbt/context/provider.py -# MacroContext -- core/dbt/context/provider.py -# ModelContext -- core/dbt/context/provider.py -# TestContext -- core/dbt/context/provider.py +# See the `contexts` module README for more information on how contexts work def get_pytz_module_context() -> Dict[str, Any]: diff --git a/core/dbt/context/secret.py b/core/dbt/context/secret.py index 5a53f2d4786..3131548a1b1 100644 --- a/core/dbt/context/secret.py +++ b/core/dbt/context/secret.py @@ -7,6 +7,9 @@ from dbt.logger import SECRET_ENV_PREFIX +SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$" + + class SecretContext(BaseContext): """This context is used in profiles.yml + packages.yml. It can render secret env vars that aren't usable elsewhere""" @@ -18,21 +21,29 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: If the default is None, raise an exception for an undefined variable. - In this context *only*, env_var will return the actual values of - env vars prefixed with DBT_ENV_SECRET_ + In this context *only*, env_var will accept env vars prefixed with DBT_ENV_SECRET_. + It will return the name of the secret env var, wrapped in 'start' and 'end' identifiers. + The actual value will be subbed in later in SecretRenderer.render_value() """ return_value = None - if var in os.environ: + + # if this is a 'secret' env var, just return the name of the env var + # instead of rendering the actual value here, to avoid any risk of + # Jinja manipulation. it will be subbed out later, in SecretRenderer.render_value + if var in os.environ and var.startswith(SECRET_ENV_PREFIX): + return SECRET_PLACEHOLDER.format(var) + + elif var in os.environ: return_value = os.environ[var] elif default is not None: return_value = default if return_value is not None: - # do not save secret environment variables + # store env vars in the internal manifest to power partial parsing + # if it's a 'secret' env var, we shouldn't even get here + # but just to be safe — don't save secrets if not var.startswith(SECRET_ENV_PREFIX): self.env_vars[var] = return_value - - # return the value even if its a secret return return_value else: msg = f"Env var required but not provided: '{var}'"