From 7b9ae20ac1f31091247213f7ac7566aca9dbd90a Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 28 Jun 2022 17:26:45 +0200 Subject: [PATCH] Backport #5382 to 1.0.latest (#5414) * Backport #5382 to 1.0.latest * Rm changelog entry for previous change * Satisfy old flake rules --- .../Under the Hood-20220616-120516.yaml | 7 ++++++ .changie.yaml | 1 + core/dbt/config/renderer.py | 7 ++++++ .../test_context_vars.py | 23 +++++++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20220616-120516.yaml diff --git a/.changes/unreleased/Under the Hood-20220616-120516.yaml b/.changes/unreleased/Under the Hood-20220616-120516.yaml new file mode 100644 index 00000000000..adf1c8b6d15 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20220616-120516.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: 'Add annotation to render_value method reimplemented in #5334' +time: 2022-06-16T12:05:16.474078+02:00 +custom: + Author: jtcohen6 + Issue: "4796" + PR: "5382" diff --git a/.changie.yaml b/.changie.yaml index d354adc1323..78e54ef0e66 100755 --- a/.changie.yaml +++ b/.changie.yaml @@ -14,6 +14,7 @@ kinds: - label: Breaking Changes - label: Docs - label: Dependencies +- label: Security custom: - key: Author label: GitHub Username(s) (separated by a single space if multiple) diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index 2977046fb38..74c6487d289 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -189,7 +189,14 @@ def name(self): return 'Secret' def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any: + # First standard Jinja rendering with special handling for 'secret' environment variables + # "{{ env_var('DBT_SECRET_ENV_VAR') }}" -> + # "$$$DBT_SECRET_START$$$DBT_SECRET_ENV_{VARIABLE_NAME}$$$DBT_SECRET_END$$$" + # This prevents Jinja manipulation of secrets via macros/filters + # that might leak partial/modified values in logs rendered = super().render_value(value, keypath) + # Now, detect instances of the placeholder value ($$$DBT_SECRET_START...DBT_SECRET_END$$$) + # and replace them with the actual secret value if SECRET_ENV_PREFIX in str(rendered): search_group = f"({SECRET_ENV_PREFIX}(.*))" pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$") diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index aac480065d3..2d7c26bfb91 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -251,6 +251,29 @@ def test_postgres_fail_clone_with_scrubbing(self): assert "abc123" not in str(exc.exception) +class TestCloneFailSecretNotRendered(TestCloneFailSecretScrubbed): + + @property + def packages_config(self): + return { + "packages": [ + { + "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') | join(' ') }}@github.com/dbt-labs/fake-repo.git" + }, + ] + } + + @use_profile('postgres') + def test_postgres_fail_clone_with_scrubbing(self): + with self.assertRaises(dbt.exceptions.InternalException) as exc: + _, log_output = self.run_dbt_and_capture(["deps"]) + + # we should not see any manipulated form of the secret value (abc123) here + # we should see a manipulated form of the placeholder instead + assert "a b c 1 2 3" not in str(exc.exception) + assert "D B T _ E N V _ S E C R E T _ G I T _ T O K E N" in str(exc.exception) + + class TestEmitWarning(DBTIntegrationTest): @property def schema(self):