From 9085458278339175a0350f2ef092eaf066b5f477 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Fri, 6 Jan 2023 17:45:52 -0800 Subject: [PATCH 01/17] Code cleanup and adding stderr to capture dbt --- core/dbt/tests/util.py | 22 ++++++++++----- tests/functional/profiles/test_profile_dir.py | 28 ++++++++----------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 3904a90a37d..a159b605d80 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -6,7 +6,10 @@ import warnings from datetime import datetime from typing import Dict, List -from contextlib import contextmanager +from contextlib import ( + contextmanager, + redirect_stdout, +) from dbt.adapters.factory import Adapter from dbt.main import handle_and_check @@ -86,22 +89,27 @@ def run_dbt(args: List[str] = None, expect_pass=True): # If you want the logs that are normally written to a file, you must # start with the "--debug" flag. The structured schema log CI test # will turn the logs into json, so you have to be prepared for that. -def run_dbt_and_capture(args: List[str] = None, expect_pass=True): +def run_dbt_and_capture(args: List[str] = None, expect_pass=True, capture_stderr=False): try: stringbuf = StringIO() capture_stdout_logs(stringbuf) - res = run_dbt(args, expect_pass=expect_pass) - stdout = stringbuf.getvalue() + if capture_stderr: + with redirect_stdout(stringbuf): + res = run_dbt(args, expect_pass=expect_pass) + else: + res = run_dbt(args, expect_pass=expect_pass) + + output = stringbuf.getvalue() finally: stop_capture_stdout_logs() # Json logs will have lots of escape characters which will # make checks for strings in the logs fail, so remove those. - if '{"code":' in stdout: - stdout = stdout.replace("\\", "") + if '{"code":' in output: + output = output.replace("\\", "") - return res, stdout + return res, output # Used in test cases to get the manifest from the partial parsing file diff --git a/tests/functional/profiles/test_profile_dir.py b/tests/functional/profiles/test_profile_dir.py index 7a4c8214a1d..7ccf0e3e47d 100644 --- a/tests/functional/profiles/test_profile_dir.py +++ b/tests/functional/profiles/test_profile_dir.py @@ -1,13 +1,17 @@ -import io import os import pytest import yaml -from contextlib import contextmanager, redirect_stdout + +from contextlib import contextmanager from pathlib import Path -from typing import List import dbt.flags as flags -from dbt.tests.util import run_dbt, write_file, rm_file + +from dbt.tests.util import ( + run_dbt_and_capture, + write_file, + rm_file, +) @pytest.fixture(scope="class") @@ -84,18 +88,8 @@ def environ(env): os.environ[key] = value -# Use this if you need to capture the standard out in a test -def run_dbt_and_capture_stdout(args: List[str] = None, expect_pass=True): - stringbuf = io.StringIO() - with redirect_stdout(stringbuf): - res = run_dbt(args, expect_pass=expect_pass) - stdout = stringbuf.getvalue() - - return res, stdout - - class TestProfiles: - def dbt_debug(self, project_dir_cli_arg=None, profiles_dir_cli_arg=None): + def dbt_debug(self, project, project_dir_cli_arg=None, profiles_dir_cli_arg=None): # begin with no command-line args or user config (from profiles.yml) flags.set_from_args({}, {}) command = ["debug"] @@ -106,8 +100,8 @@ def dbt_debug(self, project_dir_cli_arg=None, profiles_dir_cli_arg=None): if profiles_dir_cli_arg: command.extend(["--profiles-dir", str(profiles_dir_cli_arg)]) - # get the output of `dbt debug` regarless of the exit code - return run_dbt_and_capture_stdout(command, expect_pass=None) + # get the output of `dbt debug` regardless of the exit code + return run_dbt_and_capture(command, expect_pass=False, capture_stderr=True) @pytest.mark.parametrize( "project_dir_cli_arg, working_directory", From 10f8b939c9995daf1d96443749d957292304e6f2 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 00:00:45 -0800 Subject: [PATCH 02/17] Debug with --log-format json now prints structured logs. --- core/dbt/events/proto_types.py | 104 +++++++++++++++++++++++++++++ core/dbt/events/types.proto | 79 ++++++++++++++++++++++ core/dbt/events/types.py | 117 +++++++++++++++++++++++++++++++++ core/dbt/task/debug.py | 97 ++++++++++++++++++++------- 4 files changed, 372 insertions(+), 25 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index bd886243295..52a41b35a46 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1636,6 +1636,110 @@ class NoNodesSelected(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) +@dataclass +class DebugEnvironmentDetails(betterproto.Message): + """Q039""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugDependenciesDetails(betterproto.Message): + """Q040""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugDependenciesSuccess(betterproto.Message): + """Q041""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugDependenciesFailure(betterproto.Message): + """Q042""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConfigurationDetails(betterproto.Message): + """Q043""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConfigurationSuccess(betterproto.Message): + """Q044""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConfigurationFailure(betterproto.Message): + """QO45""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConnectionDetails(betterproto.Message): + """Q046""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConnectionSuccess(betterproto.Message): + """Q047""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConnectionFailure(betterproto.Message): + """Q048""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugRunSuccess(betterproto.Message): + """Q049""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugRunFailure(betterproto.Message): + """Q050""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugMiscMessages(betterproto.Message): + """Q051""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + @dataclass class CatchableExceptionOnRun(betterproto.Message): """W002""" diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index ec10b906432..fdd2369248b 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1289,6 +1289,85 @@ message NoNodesSelected { EventInfo info = 1; } +// Q039 +message DebugEnvironmentDetails { + EventInfo info = 1; + string msg = 2; +} + +// Q040 +message DebugDependenciesDetails { + EventInfo info = 1; + string msg = 2; +} + +// Q041 +message DebugDependenciesSuccess { + EventInfo info = 1; + string msg = 2; +} + +// Q042 +message DebugDependenciesFailure { + EventInfo info = 1; + string msg = 2; +} + +// Q043 +message DebugConfigurationDetails { + EventInfo info = 1; + string msg = 2; +} + +// Q044 +message DebugConfigurationSuccess { + EventInfo info = 1; + string msg = 2; +} + +// QO45 +message DebugConfigurationFailure { + EventInfo info = 1; + string msg = 2; +} + +// Q046 +message DebugConnectionDetails { + EventInfo info = 1; + string msg = 2; +} + +// Q047 +message DebugConnectionSuccess { + EventInfo info = 1; + string msg = 2; +} + +// Q048 +message DebugConnectionFailure { + EventInfo info = 1; + string msg = 2; +} + +// Q049 +message DebugRunSuccess { + EventInfo info = 1; + string msg = 2; +} + +// Q050 +message DebugRunFailure { + EventInfo info = 1; + string msg = 2; +} + +// Q051 +message DebugMiscMessages { + EventInfo info = 1; + string msg = 2; +} + + // W - Node testing // Skipped W001 diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 6a597184cd7..e83faa42d4e 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2039,6 +2039,123 @@ def message(self) -> str: return "No nodes selected!" +@dataclass +class DebugEnvironmentDetails(InfoLevel, pt.DebugEnvironmentDetails): + def code(self): + return "Q039" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugDependenciesDetails(InfoLevel, pt.DebugDependenciesDetails): + def code(self): + return "Q040" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugDependenciesSuccess(InfoLevel, pt.DebugDependenciesSuccess): + def code(self): + return "Q041" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugDependenciesFailure(ErrorLevel, pt.DebugDependenciesFailure): + def code(self): + return "Q042" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConfigurationDetails(InfoLevel, pt.DebugConfigurationDetails): + def code(self): + return "Q043" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConfigurationSuccess(InfoLevel, pt.DebugConfigurationSuccess): + def code(self): + return "Q044" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConfigurationFailure(ErrorLevel, pt.DebugConfigurationFailure): + def code(self): + return "Q045" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConnectionDetails(ErrorLevel, pt.DebugConnectionDetails): + def code(self): + return "Q046" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConnectionSuccess(ErrorLevel, pt.DebugConnectionSuccess): + def code(self): + return "Q047" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConnectionFailure(ErrorLevel, pt.DebugConnectionFailure): + def code(self): + return "Q048" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugRunSuccess(ErrorLevel, pt.DebugRunSuccess): + def code(self): + return "Q049" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugRunFailure(ErrorLevel, pt.DebugRunFailure): + def code(self): + return "Q050" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugMiscMessages(ErrorLevel, pt.DebugMiscMessages): + def code(self): + return "Q051" + + def message(self) -> str: + return self.msg + + # ======================================================= # W - Node testing # ======================================================= diff --git a/core/dbt/task/debug.py b/core/dbt/task/debug.py index 5f3e3854759..26a05489905 100644 --- a/core/dbt/task/debug.py +++ b/core/dbt/task/debug.py @@ -5,7 +5,22 @@ from typing import Optional, Dict, Any, List from dbt.events.functions import fire_event -from dbt.events.types import OpenCommand +from dbt.events.types import ( + OpenCommand, + DebugEnvironmentDetails, + DebugDependenciesDetails, + DebugDependenciesSuccess, + DebugDependenciesFailure, + DebugConfigurationDetails, + DebugConfigurationSuccess, + DebugConfigurationFailure, + DebugConnectionDetails, + DebugConnectionSuccess, + DebugConnectionFailure, + DebugRunSuccess, + DebugRunFailure, + DebugMiscMessages, +) from dbt import flags import dbt.clients.system import dbt.exceptions @@ -99,25 +114,33 @@ def run(self): return not self.any_failure version = get_installed_version().to_version_string(skip_matcher=True) - print("dbt version: {}".format(version)) - print("python version: {}".format(sys.version.split()[0])) - print("python path: {}".format(sys.executable)) - print("os info: {}".format(platform.platform())) - print("Using profiles.yml file at {}".format(self.profile_path)) - print("Using dbt_project.yml file at {}".format(self.project_path)) - print("") + fire_event(DebugEnvironmentDetails(msg="dbt version: {}".format(version))) + fire_event( + DebugEnvironmentDetails(msg="python version: {}".format(sys.version.split()[0])) + ) + fire_event(DebugEnvironmentDetails(msg="python path: {}".format(sys.executable))) + fire_event(DebugEnvironmentDetails(msg="os info: {}".format(platform.platform()))) + fire_event( + DebugEnvironmentDetails(msg="Using profiles.yml file at {}".format(self.profile_path)) + ) + fire_event( + DebugEnvironmentDetails( + msg="Using dbt_project.yml file at {}".format(self.project_path) + ) + ) self.test_configuration() self.test_dependencies() self.test_connection() if self.any_failure: - print(red(f"{(pluralize(len(self.messages), 'check'))} failed:")) + fire_event( + DebugRunFailure(msg=red(f"{(pluralize(len(self.messages), 'check'))} failed:")) + ) else: - print(green("All checks passed!")) + fire_event(DebugRunSuccess(msg=green("All checks passed!"))) for message in self.messages: - print(message) - print("") + fire_event(DebugMiscMessages(msg=f"{message}\n")) return not self.any_failure @@ -273,21 +296,43 @@ def test_git(self): return green("OK found") def test_dependencies(self): - print("Required dependencies:") - print(" - git [{}]".format(self.test_git())) - print("") + fire_event(DebugDependenciesDetails(msg="Required dependencies:")) + + logline_msg = self.test_git() + event_type = DebugDependenciesSuccess if "OK" in logline_msg else DebugDependenciesFailure + fire_event(event_type(msg=f" - git [{logline_msg}]\n")) def test_configuration(self): + def get_config_result_type(status): + return DebugConfigurationSuccess if "OK" in status else DebugConfigurationFailure + + fire_event(DebugConfigurationDetails(msg="Configuration:")) + profile_status = self._load_profile() + fire_event( + get_config_result_type(profile_status)(msg=f" profiles.yml file [{profile_status}]") + ) + project_status = self._load_project() - print("Configuration:") - print(" profiles.yml file [{}]".format(profile_status)) - print(" dbt_project.yml file [{}]".format(project_status)) + fire_event( + get_config_result_type(project_status)( + msg=f" dbt_project.yml file [{project_status}]" + ) + ) + # skip profile stuff if we can't find a profile name if self.profile_name is not None: - print(" profile: {} [{}]".format(self.profile_name, self._profile_found())) - print(" target: {} [{}]".format(self.target_name, self._target_found())) - print("") + fire_event( + DebugConfigurationDetails( + msg=" profile: {} [{}]\n".format(self.profile_name, self._profile_found()) + ) + ) + fire_event( + DebugConfigurationDetails( + msg=" target: {} [{}]\n".format(self.target_name, self._target_found()) + ) + ) + self._log_project_fail() self._log_profile_fail() @@ -348,11 +393,13 @@ def _connection_result(self): def test_connection(self): if not self.profile: return - print("Connection:") + fire_event(DebugConnectionDetails(msg="Connection:")) for k, v in self.profile.credentials.connection_info(): - print(" {}: {}".format(k, v)) - print(" Connection test: [{}]".format(self._connection_result())) - print("") + fire_event(DebugConnectionDetails(msg=f" {k}: {v}")) + + res = self._connection_result() + event_type = DebugConnectionSuccess if "OK" in res else DebugConnectionFailure + fire_event(event_type(msg=f" Connection test: [{res}]\n")) @classmethod def validate_connection(cls, target_dict): From 17fecdc10590c5c314d5402e64ee7dcaab4b63ba Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 00:32:38 -0800 Subject: [PATCH 03/17] Add changelog. --- .changes/unreleased/Features-20230107-003157.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230107-003157.yaml diff --git a/.changes/unreleased/Features-20230107-003157.yaml b/.changes/unreleased/Features-20230107-003157.yaml new file mode 100644 index 00000000000..27858b516be --- /dev/null +++ b/.changes/unreleased/Features-20230107-003157.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Have dbt debug spit out structured json logs with flags enabled. +time: 2023-01-07T00:31:57.516063-08:00 +custom: + Author: versusfacit + Issue: "5353" From d1a420ae45f9d226710cd4940284a3502fc436bb Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 00:53:06 -0800 Subject: [PATCH 04/17] Move logs into miscellaneous and add values to test. --- core/dbt/events/proto_types.py | 208 ++++++++++++++--------------- core/dbt/events/types.proto | 157 +++++++++++----------- core/dbt/events/types.py | 234 ++++++++++++++++----------------- tests/unit/test_events.py | 13 ++ 4 files changed, 313 insertions(+), 299 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 52a41b35a46..49e8f44e61b 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1636,110 +1636,6 @@ class NoNodesSelected(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) -@dataclass -class DebugEnvironmentDetails(betterproto.Message): - """Q039""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugDependenciesDetails(betterproto.Message): - """Q040""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugDependenciesSuccess(betterproto.Message): - """Q041""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugDependenciesFailure(betterproto.Message): - """Q042""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugConfigurationDetails(betterproto.Message): - """Q043""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugConfigurationSuccess(betterproto.Message): - """Q044""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugConfigurationFailure(betterproto.Message): - """QO45""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugConnectionDetails(betterproto.Message): - """Q046""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugConnectionSuccess(betterproto.Message): - """Q047""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugConnectionFailure(betterproto.Message): - """Q048""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugRunSuccess(betterproto.Message): - """Q049""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugRunFailure(betterproto.Message): - """Q050""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - -@dataclass -class DebugMiscMessages(betterproto.Message): - """Q051""" - - info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) - - @dataclass class CatchableExceptionOnRun(betterproto.Message): """W002""" @@ -2120,6 +2016,110 @@ class RunResultWarningMessage(betterproto.Message): msg: str = betterproto.string_field(2) +@dataclass +class DebugEnvironmentDetails(betterproto.Message): + """Z047""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugDependenciesDetails(betterproto.Message): + """Z048""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugDependenciesSuccess(betterproto.Message): + """Z049""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugDependenciesFailure(betterproto.Message): + """Z050""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConfigurationDetails(betterproto.Message): + """Z051""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConfigurationSuccess(betterproto.Message): + """Z052""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConfigurationFailure(betterproto.Message): + """QO53""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConnectionDetails(betterproto.Message): + """Z054""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConnectionSuccess(betterproto.Message): + """Z055""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugConnectionFailure(betterproto.Message): + """Z056""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugRunSuccess(betterproto.Message): + """Z057""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugRunFailure(betterproto.Message): + """Z058""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + +@dataclass +class DebugMiscMessages(betterproto.Message): + """Z059""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + @dataclass class IntegrationTestInfo(betterproto.Message): """T001""" diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index fdd2369248b..9e758f9eb25 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1289,84 +1289,6 @@ message NoNodesSelected { EventInfo info = 1; } -// Q039 -message DebugEnvironmentDetails { - EventInfo info = 1; - string msg = 2; -} - -// Q040 -message DebugDependenciesDetails { - EventInfo info = 1; - string msg = 2; -} - -// Q041 -message DebugDependenciesSuccess { - EventInfo info = 1; - string msg = 2; -} - -// Q042 -message DebugDependenciesFailure { - EventInfo info = 1; - string msg = 2; -} - -// Q043 -message DebugConfigurationDetails { - EventInfo info = 1; - string msg = 2; -} - -// Q044 -message DebugConfigurationSuccess { - EventInfo info = 1; - string msg = 2; -} - -// QO45 -message DebugConfigurationFailure { - EventInfo info = 1; - string msg = 2; -} - -// Q046 -message DebugConnectionDetails { - EventInfo info = 1; - string msg = 2; -} - -// Q047 -message DebugConnectionSuccess { - EventInfo info = 1; - string msg = 2; -} - -// Q048 -message DebugConnectionFailure { - EventInfo info = 1; - string msg = 2; -} - -// Q049 -message DebugRunSuccess { - EventInfo info = 1; - string msg = 2; -} - -// Q050 -message DebugRunFailure { - EventInfo info = 1; - string msg = 2; -} - -// Q051 -message DebugMiscMessages { - EventInfo info = 1; - string msg = 2; -} - // W - Node testing @@ -1664,6 +1586,85 @@ message RunResultWarningMessage { string msg = 2; } +// Z047 +message DebugEnvironmentDetails { + EventInfo info = 1; + string msg = 2; +} + +// Z048 +message DebugDependenciesDetails { + EventInfo info = 1; + string msg = 2; +} + +// Z049 +message DebugDependenciesSuccess { + EventInfo info = 1; + string msg = 2; +} + +// Z050 +message DebugDependenciesFailure { + EventInfo info = 1; + string msg = 2; +} + +// Z051 +message DebugConfigurationDetails { + EventInfo info = 1; + string msg = 2; +} + +// Z052 +message DebugConfigurationSuccess { + EventInfo info = 1; + string msg = 2; +} + +// QO53 +message DebugConfigurationFailure { + EventInfo info = 1; + string msg = 2; +} + +// Z054 +message DebugConnectionDetails { + EventInfo info = 1; + string msg = 2; +} + +// Z055 +message DebugConnectionSuccess { + EventInfo info = 1; + string msg = 2; +} + +// Z056 +message DebugConnectionFailure { + EventInfo info = 1; + string msg = 2; +} + +// Z057 +message DebugRunSuccess { + EventInfo info = 1; + string msg = 2; +} + +// Z058 +message DebugRunFailure { + EventInfo info = 1; + string msg = 2; +} + +// Z059 +message DebugMiscMessages { + EventInfo info = 1; + string msg = 2; +} + + // T - Integration tests // T001 diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index e83faa42d4e..6c35abef59d 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2039,123 +2039,6 @@ def message(self) -> str: return "No nodes selected!" -@dataclass -class DebugEnvironmentDetails(InfoLevel, pt.DebugEnvironmentDetails): - def code(self): - return "Q039" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugDependenciesDetails(InfoLevel, pt.DebugDependenciesDetails): - def code(self): - return "Q040" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugDependenciesSuccess(InfoLevel, pt.DebugDependenciesSuccess): - def code(self): - return "Q041" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugDependenciesFailure(ErrorLevel, pt.DebugDependenciesFailure): - def code(self): - return "Q042" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConfigurationDetails(InfoLevel, pt.DebugConfigurationDetails): - def code(self): - return "Q043" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConfigurationSuccess(InfoLevel, pt.DebugConfigurationSuccess): - def code(self): - return "Q044" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConfigurationFailure(ErrorLevel, pt.DebugConfigurationFailure): - def code(self): - return "Q045" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConnectionDetails(ErrorLevel, pt.DebugConnectionDetails): - def code(self): - return "Q046" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConnectionSuccess(ErrorLevel, pt.DebugConnectionSuccess): - def code(self): - return "Q047" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConnectionFailure(ErrorLevel, pt.DebugConnectionFailure): - def code(self): - return "Q048" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugRunSuccess(ErrorLevel, pt.DebugRunSuccess): - def code(self): - return "Q049" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugRunFailure(ErrorLevel, pt.DebugRunFailure): - def code(self): - return "Q050" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugMiscMessages(ErrorLevel, pt.DebugMiscMessages): - def code(self): - return "Q051" - - def message(self) -> str: - return self.msg - - # ======================================================= # W - Node testing # ======================================================= @@ -2639,3 +2522,120 @@ def code(self): def message(self) -> str: # This is the message on the result object, cannot be formatted in event return self.msg + + +@dataclass +class DebugEnvironmentDetails(InfoLevel, pt.DebugEnvironmentDetails): + def code(self): + return "Z047" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugDependenciesDetails(InfoLevel, pt.DebugDependenciesDetails): + def code(self): + return "Z048" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugDependenciesSuccess(InfoLevel, pt.DebugDependenciesSuccess): + def code(self): + return "Z049" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugDependenciesFailure(ErrorLevel, pt.DebugDependenciesFailure): + def code(self): + return "Z050" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConfigurationDetails(InfoLevel, pt.DebugConfigurationDetails): + def code(self): + return "Z051" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConfigurationSuccess(InfoLevel, pt.DebugConfigurationSuccess): + def code(self): + return "Z052" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConfigurationFailure(ErrorLevel, pt.DebugConfigurationFailure): + def code(self): + return "Z053" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConnectionDetails(ErrorLevel, pt.DebugConnectionDetails): + def code(self): + return "Z054" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConnectionSuccess(ErrorLevel, pt.DebugConnectionSuccess): + def code(self): + return "Z055" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugConnectionFailure(ErrorLevel, pt.DebugConnectionFailure): + def code(self): + return "Z056" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugRunSuccess(ErrorLevel, pt.DebugRunSuccess): + def code(self): + return "Z057" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugRunFailure(ErrorLevel, pt.DebugRunFailure): + def code(self): + return "Z058" + + def message(self) -> str: + return self.msg + + +@dataclass +class DebugMiscMessages(ErrorLevel, pt.DebugMiscMessages): + def code(self): + return "Z059" + + def message(self) -> str: + return self.msg diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index e37d26ad552..d8e81e161bc 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -464,6 +464,19 @@ def MockNode(): FlushEventsFailure(), TrackingInitializeFailure(), RunResultWarningMessage(), + DebugEnvironmentDetails(msg=""), + DebugDependenciesDetails(msg=""), + DebugDependenciesSuccess(msg=""), + DebugDependenciesFailure(msg=""), + DebugConfigurationDetails(msg=""), + DebugConfigurationSuccess(msg=""), + DebugConfigurationFailure(msg=""), + DebugConnectionDetails(msg=""), + DebugConnectionSuccess(msg=""), + DebugConnectionFailure(msg=""), + DebugRunSuccess(msg=""), + DebugRunFailure(msg=""), + DebugMiscMessages(msg=""), # T - tests ====================== IntegrationTestInfo(), From 39c3c36e41bad566a86d0f7f2cc2185c7282521e Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 01:00:30 -0800 Subject: [PATCH 05/17] nix whitespace and fix log levels --- core/dbt/events/types.proto | 1 - core/dbt/events/types.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 9e758f9eb25..95fb467fbf2 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1289,7 +1289,6 @@ message NoNodesSelected { EventInfo info = 1; } - // W - Node testing // Skipped W001 diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 6c35abef59d..91461e6c711 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2588,7 +2588,7 @@ def message(self) -> str: @dataclass -class DebugConnectionDetails(ErrorLevel, pt.DebugConnectionDetails): +class DebugConnectionDetails(InfoLevel, pt.DebugConnectionDetails): def code(self): return "Z054" @@ -2597,7 +2597,7 @@ def message(self) -> str: @dataclass -class DebugConnectionSuccess(ErrorLevel, pt.DebugConnectionSuccess): +class DebugConnectionSuccess(InfoLevel, pt.DebugConnectionSuccess): def code(self): return "Z055" @@ -2615,7 +2615,7 @@ def message(self) -> str: @dataclass -class DebugRunSuccess(ErrorLevel, pt.DebugRunSuccess): +class DebugRunSuccess(InfoLevel, pt.DebugRunSuccess): def code(self): return "Z057" @@ -2633,7 +2633,7 @@ def message(self) -> str: @dataclass -class DebugMiscMessages(ErrorLevel, pt.DebugMiscMessages): +class DebugMiscMessages(WarnLevel, pt.DebugMiscMessages): def code(self): return "Z059" From 65a01e15b043a69d33c4a5e7c13b8ffb0b61486d Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 18:09:02 -0800 Subject: [PATCH 06/17] List will now do structured logging when log format set to json. --- core/dbt/compilation.py | 5 ++++- core/dbt/events/proto_types.py | 8 ++++++++ core/dbt/events/types.proto | 6 ++++++ core/dbt/events/types.py | 9 +++++++++ core/dbt/main.py | 14 +++++++------- core/dbt/task/list.py | 34 +++++++++++++++------------------- tests/unit/test_events.py | 1 + 7 files changed, 50 insertions(+), 27 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 4ae78fd3485..cf0d5dfdd8d 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -32,6 +32,7 @@ from dbt.node_types import NodeType, ModelLanguage from dbt.events.format import pluralize import dbt.tracking +import dbt.task.list as list_task graph_file_name = "graph.gpickle" @@ -482,7 +483,9 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph if write: self.write_graph_file(linker, manifest) - print_compile_stats(stats) + + if not list_task.ListTask == self.config.args.cls: + print_compile_stats(stats) return Graph(linker.graph) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 49e8f44e61b..70dc7bd240b 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -2120,6 +2120,14 @@ class DebugMiscMessages(betterproto.Message): msg: str = betterproto.string_field(2) +@dataclass +class ListRunDetails(betterproto.Message): + """Z060""" + + info: "EventInfo" = betterproto.message_field(1) + msg: str = betterproto.string_field(2) + + @dataclass class IntegrationTestInfo(betterproto.Message): """T001""" diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 95fb467fbf2..fb3960a2530 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1663,6 +1663,12 @@ message DebugMiscMessages { string msg = 2; } +// Z060 +message ListRunDetails { + EventInfo info = 1; + string msg = 2; +} + // T - Integration tests diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 91461e6c711..6d001ed7716 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2639,3 +2639,12 @@ def code(self): def message(self) -> str: return self.msg + + +@dataclass +class ListRunDetails(InfoLevel, pt.ListRunDetails): + def code(self): + return "Z060" + + def message(self) -> str: + return self.msg diff --git a/core/dbt/main.py b/core/dbt/main.py index 55920e8a5cc..197f20fc8c0 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -229,15 +229,15 @@ def run_from_args(parsed): if task.config is not None: log_path = getattr(task.config, "log_path", None) log_manager.set_path(log_path) - # if 'list' task: set stdout to WARN instead of INFO - level_override = parsed.cls.pre_init_hook(parsed) - setup_event_logger(log_path or "logs", level_override) + setup_event_logger(log_path or "logs") - fire_event(MainReportVersion(version=str(dbt.version.installed), log_version=LOG_VERSION)) - fire_event(MainReportArgs(args=args_to_dict(parsed))) + # For the ListTask, filter out system report logs to allow piping ls output to jq, etc + if not list_task.ListTask == parsed.cls: + fire_event(MainReportVersion(version=str(dbt.version.installed), log_version=LOG_VERSION)) + fire_event(MainReportArgs(args=args_to_dict(parsed))) - if dbt.tracking.active_user is not None: # mypy appeasement, always true - fire_event(MainTrackingUserState(user_state=dbt.tracking.active_user.state())) + if dbt.tracking.active_user is not None: # mypy appeasement, always true + fire_event(MainTrackingUserState(user_state=dbt.tracking.active_user.state())) results = None diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index e1be8f214d3..d5ffee2337c 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -1,15 +1,20 @@ import json from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric +from dbt.flags import get_flag_dict from dbt.graph import ResourceTypeSelector from dbt.task.runnable import GraphRunnableTask, ManifestTask from dbt.task.test import TestSelector from dbt.node_types import NodeType -from dbt.events.functions import warn_or_error -from dbt.events.types import NoNodesSelected +from dbt.events.functions import ( + fire_event, + warn_or_error, +) +from dbt.events.types import ( + NoNodesSelected, + ListRunDetails, +) from dbt.exceptions import RuntimeException, InternalException -from dbt.logger import log_manager -from dbt.events.eventmgr import EventLevel class ListTask(GraphRunnableTask): @@ -50,20 +55,6 @@ def __init__(self, args, config): '"models" and "resource_type" are mutually exclusive ' "arguments" ) - @classmethod - def pre_init_hook(cls, args): - """A hook called before the task is initialized.""" - # Filter out all INFO-level logging to allow piping ls output to jq, etc - # WARN level will still include all warnings + errors - # Do this by: - # - returning the log level so that we can pass it into the 'level_override' - # arg of events.functions.setup_event_logger() -- good! - # - mutating the initialized, not-yet-configured STDOUT event logger - # because it's being configured too late -- bad! TODO refactor! - log_manager.stderr_console() - super().pre_init_hook(args) - return EventLevel.WARN - def _iterate_selected_nodes(self): selector = self.get_node_selector() spec = self.get_selection_spec() @@ -148,9 +139,14 @@ def run(self): return self.output_results(generator()) def output_results(self, results): + """Log, or output a plain, newline-delimited, and ready-to-pipe list of nodes found.""" for result in results: self.node_results.append(result) - print(result) + if get_flag_dict()["log_format"] == "json": + fire_event(ListRunDetails(msg=result)) + else: + # Cleaner to leave as print than to mutate the logger. + print(result) return self.node_results @property diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index d8e81e161bc..3cdaba76f55 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -477,6 +477,7 @@ def MockNode(): DebugRunSuccess(msg=""), DebugRunFailure(msg=""), DebugMiscMessages(msg=""), + ListRunDetails(msg=""), # T - tests ====================== IntegrationTestInfo(), From 3505aae514b94d57dee746de46ef5cf3d0cbe65d Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 18:19:36 -0800 Subject: [PATCH 07/17] Add a quick None check. --- core/dbt/compilation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index cf0d5dfdd8d..d537718490d 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -484,7 +484,7 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph if write: self.write_graph_file(linker, manifest) - if not list_task.ListTask == self.config.args.cls: + if self.config.args and not list_task.ListTask == self.config.args.cls: print_compile_stats(stats) return Graph(linker.graph) From ec6c9dde92200aa456d428aaca88f7609e478331 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 18:32:53 -0800 Subject: [PATCH 08/17] Add a get guard to class check. --- core/dbt/compilation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index d537718490d..af8ec9a32e9 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -484,7 +484,7 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph if write: self.write_graph_file(linker, manifest) - if self.config.args and not list_task.ListTask == self.config.args.cls: + if self.config.args and not list_task.ListTask == self.config.args.get("cls", None): print_compile_stats(stats) return Graph(linker.graph) From ee24e046971289134fb825432d3ca4b5e757a842 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 18:43:04 -0800 Subject: [PATCH 09/17] Better null checking --- core/dbt/compilation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index af8ec9a32e9..24888c24b9b 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -484,7 +484,9 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph if write: self.write_graph_file(linker, manifest) - if self.config.args and not list_task.ListTask == self.config.args.get("cls", None): + if getattr( + self.config.args, "get", None + ) and not list_task.ListTask == self.config.args.get("cls", None): print_compile_stats(stats) return Graph(linker.graph) From de12d6418b715c5413b76116a180407657798475 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Sat, 7 Jan 2023 18:59:08 -0800 Subject: [PATCH 10/17] The boolean doesn't reflect the original logic but a try-catch does. --- core/dbt/compilation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 24888c24b9b..563b4bb1171 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -484,9 +484,11 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph if write: self.write_graph_file(linker, manifest) - if getattr( - self.config.args, "get", None - ) and not list_task.ListTask == self.config.args.get("cls", None): + # Only in the case the object is specifically a ListTask do we avoid printing these stats. + try: + if not isinstance(self.config.args.get("cls", None), list_task.ListTask): + print_compile_stats(stats) + except AttributeError: print_compile_stats(stats) return Graph(linker.graph) From c8e03eb7703ea8b813f7e9211a52c4ff3f1058ed Mon Sep 17 00:00:00 2001 From: Mila Page Date: Thu, 19 Jan 2023 19:07:45 -0800 Subject: [PATCH 11/17] Address some code review comments and get us working again. --- core/dbt/events/proto_types.py | 100 +++++++++++++++--- core/dbt/events/types.proto | 84 ++++++++++++--- core/dbt/tests/util.py | 22 ++-- tests/functional/profiles/test_profile_dir.py | 2 +- tests/unit/test_events.py | 29 +++-- 5 files changed, 177 insertions(+), 60 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 67e489f6798..0ed5ff93939 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1,5 +1,5 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! -# sources: types.proto +# sources: core/dbt/events/types.proto # plugin: python-betterproto from dataclasses import dataclass from datetime import datetime @@ -2851,112 +2851,182 @@ class RunResultWarningMessageMsg(betterproto.Message): class DebugEnvironmentDetails(betterproto.Message): """Z047""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugEnvironmentDetailsMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugEnvironmentDetails" = betterproto.message_field(2) @dataclass class DebugDependenciesDetails(betterproto.Message): """Z048""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugDependenciesDetailsMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugDependenciesDetails" = betterproto.message_field(2) @dataclass class DebugDependenciesSuccess(betterproto.Message): """Z049""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugDependenciesSuccessMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugDependenciesSuccess" = betterproto.message_field(2) @dataclass class DebugDependenciesFailure(betterproto.Message): """Z050""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugDependenciesFailureMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugDependenciesFailure" = betterproto.message_field(2) @dataclass class DebugConfigurationDetails(betterproto.Message): """Z051""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugConfigurationDetailsMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugConfigurationDetails" = betterproto.message_field(2) @dataclass class DebugConfigurationSuccess(betterproto.Message): """Z052""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugConfigurationSuccessMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugConfigurationSuccess" = betterproto.message_field(2) @dataclass class DebugConfigurationFailure(betterproto.Message): """QO53""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugConfigurationFailureMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugConfigurationFailure" = betterproto.message_field(2) @dataclass class DebugConnectionDetails(betterproto.Message): """Z054""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugConnectionDetailsMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugConnectionDetails" = betterproto.message_field(2) @dataclass class DebugConnectionSuccess(betterproto.Message): """Z055""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugConnectionSuccessMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugConnectionSuccess" = betterproto.message_field(2) @dataclass class DebugConnectionFailure(betterproto.Message): """Z056""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugConnectionFailureMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugConnectionFailure" = betterproto.message_field(2) @dataclass class DebugRunSuccess(betterproto.Message): """Z057""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugRunSuccessMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugRunSuccess" = betterproto.message_field(2) @dataclass class DebugRunFailure(betterproto.Message): """Z058""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugRunFailureMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugRunFailure" = betterproto.message_field(2) @dataclass class DebugMiscMessages(betterproto.Message): """Z059""" + msg: str = betterproto.string_field(1) + + +@dataclass +class DebugMiscMessagesMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "DebugMiscMessages" = betterproto.message_field(2) @dataclass class ListRunDetails(betterproto.Message): """Z060""" + msg: str = betterproto.string_field(1) + + +@dataclass +class ListRunDetailsMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - msg: str = betterproto.string_field(2) + data: "ListRunDetails" = betterproto.message_field(2) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 059eb4b2a09..da23375e8ab 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -2260,86 +2260,142 @@ message RunResultWarningMessageMsg { // Z047 message DebugEnvironmentDetails { + string msg = 1; +} + +message DebugEnvironmentDetailsMsg { EventInfo info = 1; - string msg = 2; + DebugEnvironmentDetails data = 2; } // Z048 message DebugDependenciesDetails { + string msg = 1; +} + +message DebugDependenciesDetailsMsg { EventInfo info = 1; - string msg = 2; + DebugDependenciesDetails data = 2; } // Z049 message DebugDependenciesSuccess { + string msg = 1; +} + +message DebugDependenciesSuccessMsg { EventInfo info = 1; - string msg = 2; + DebugDependenciesSuccess data = 2; } // Z050 message DebugDependenciesFailure { + string msg = 1; +} + +message DebugDependenciesFailureMsg { EventInfo info = 1; - string msg = 2; + DebugDependenciesFailure data = 2; } // Z051 message DebugConfigurationDetails { + string msg = 1; +} + +message DebugConfigurationDetailsMsg { EventInfo info = 1; - string msg = 2; + DebugConfigurationDetails data = 2; } // Z052 message DebugConfigurationSuccess { + string msg = 1; +} + +message DebugConfigurationSuccessMsg { EventInfo info = 1; - string msg = 2; + DebugConfigurationSuccess data = 2; } // QO53 message DebugConfigurationFailure { + string msg = 1; +} + +message DebugConfigurationFailureMsg { EventInfo info = 1; - string msg = 2; + DebugConfigurationFailure data = 2; } // Z054 message DebugConnectionDetails { + string msg = 1; +} + +message DebugConnectionDetailsMsg { EventInfo info = 1; - string msg = 2; + DebugConnectionDetails data = 2; } // Z055 message DebugConnectionSuccess { + string msg = 1; +} + +message DebugConnectionSuccessMsg { EventInfo info = 1; - string msg = 2; + DebugConnectionSuccess data = 2; } // Z056 message DebugConnectionFailure { + string msg = 1; +} + +message DebugConnectionFailureMsg { EventInfo info = 1; - string msg = 2; + DebugConnectionFailure data = 2; } // Z057 message DebugRunSuccess { + string msg = 1; +} + +message DebugRunSuccessMsg { EventInfo info = 1; - string msg = 2; + DebugRunSuccess data = 2; } // Z058 message DebugRunFailure { + string msg = 1; +} + +message DebugRunFailureMsg { EventInfo info = 1; - string msg = 2; + DebugRunFailure data = 2; } // Z059 message DebugMiscMessages { + string msg = 1; +} + +message DebugMiscMessagesMsg { EventInfo info = 1; - string msg = 2; + DebugMiscMessages data = 2; } // Z060 message ListRunDetails { + string msg = 1; +} + +message ListRunDetailsMsg { EventInfo info = 1; - string msg = 2; + ListRunDetails data = 2; } diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 03e752405ad..245648ceb48 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -6,10 +6,7 @@ import warnings from datetime import datetime from typing import Dict, List -from contextlib import ( - contextmanager, - redirect_stdout, -) +from contextlib import contextmanager from dbt.adapters.factory import Adapter from dbt.main import handle_and_check @@ -94,27 +91,22 @@ def run_dbt(args: List[str] = None, expect_pass=True): # If you want the logs that are normally written to a file, you must # start with the "--debug" flag. The structured schema log CI test # will turn the logs into json, so you have to be prepared for that. -def run_dbt_and_capture(args: List[str] = None, expect_pass=True, capture_stderr=False): +def run_dbt_and_capture(args: List[str] = None, expect_pass=True): try: stringbuf = StringIO() capture_stdout_logs(stringbuf) - if capture_stderr: - with redirect_stdout(stringbuf): - res = run_dbt(args, expect_pass=expect_pass) - else: - res = run_dbt(args, expect_pass=expect_pass) - - output = stringbuf.getvalue() + res = run_dbt(args, expect_pass=expect_pass) + stdout = stringbuf.getvalue() finally: stop_capture_stdout_logs() # Json logs will have lots of escape characters which will # make checks for strings in the logs fail, so remove those. - if '{"code":' in output: - output = output.replace("\\", "") + if '{"code":' in stdout: + stdout = stdout.replace("\\", "") - return res, output + return res, stdout # Used in test cases to get the manifest from the partial parsing file diff --git a/tests/functional/profiles/test_profile_dir.py b/tests/functional/profiles/test_profile_dir.py index 7ccf0e3e47d..536323655db 100644 --- a/tests/functional/profiles/test_profile_dir.py +++ b/tests/functional/profiles/test_profile_dir.py @@ -101,7 +101,7 @@ def dbt_debug(self, project, project_dir_cli_arg=None, profiles_dir_cli_arg=None command.extend(["--profiles-dir", str(profiles_dir_cli_arg)]) # get the output of `dbt debug` regardless of the exit code - return run_dbt_and_capture(command, expect_pass=False, capture_stderr=True) + return run_dbt_and_capture(command, expect_pass=None) @pytest.mark.parametrize( "project_dir_cli_arg, working_directory", diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index cc23b016762..dd4560690c6 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -383,21 +383,20 @@ def test_event_codes(self): FlushEventsFailure(), TrackingInitializeFailure(), RunResultWarningMessage(), - # TODO fix these - DebugEnvironmentDetails(msg=""), - # DebugDependenciesDetails(msg=""), - # DebugDependenciesSuccess(msg=""), - # DebugDependenciesFailure(msg=""), - # DebugConfigurationDetails(msg=""), - # DebugConfigurationSuccess(msg=""), - # DebugConfigurationFailure(msg=""), - # DebugConnectionDetails(msg=""), - # DebugConnectionSuccess(msg=""), - # DebugConnectionFailure(msg=""), - # DebugRunSuccess(msg=""), - # DebugRunFailure(msg=""), - # DebugMiscMessages(msg=""), - # ListRunDetails(msg=""), + DebugEnvironmentDetails(), + DebugDependenciesDetails(), + DebugDependenciesSuccess(), + DebugDependenciesFailure(), + DebugConfigurationDetails(), + DebugConfigurationSuccess(), + DebugConfigurationFailure(), + DebugConnectionDetails(), + DebugConnectionSuccess(), + DebugConnectionFailure(), + DebugRunSuccess(), + DebugRunFailure(), + DebugMiscMessages(), + ListRunDetails(), # T - tests ====================== IntegrationTestInfo(), IntegrationTestDebug(), From f0c5f5be0c35d57abf0402935352a4225557be62 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Thu, 19 Jan 2023 19:25:56 -0800 Subject: [PATCH 12/17] Simplify logic now that we have a namespace object for self.config.args. --- core/dbt/compilation.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 961ef03cf85..eb7d33cae34 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -475,11 +475,8 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph if write: self.write_graph_file(linker, manifest) - # Only in the case the object is specifically a ListTask do we avoid printing these stats. - try: - if not isinstance(self.config.args.get("cls", None), list_task.ListTask): - print_compile_stats(stats) - except AttributeError: + # Do not print these for ListTask's + if not self.config.args.cls == list_task.ListTask: print_compile_stats(stats) return Graph(linker.graph) From 57549eb68c834908b5a580111937fc0154d610a0 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Thu, 19 Jan 2023 19:36:03 -0800 Subject: [PATCH 13/17] Simplify logic for json log format checking. --- core/dbt/task/list.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 01f1864ccae..a2e4ffdf302 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -1,7 +1,8 @@ import json +import dbt.flags + from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric -from dbt.flags import get_flag_dict from dbt.graph import ResourceTypeSelector from dbt.task.runnable import GraphRunnableTask, ManifestTask from dbt.task.test import TestSelector @@ -142,10 +143,10 @@ def output_results(self, results): """Log, or output a plain, newline-delimited, and ready-to-pipe list of nodes found.""" for result in results: self.node_results.append(result) - if get_flag_dict()["log_format"] == "json": + if dbt.flags.LOG_FORMAT == "json": fire_event(ListRunDetails(msg=result)) else: - # Cleaner to leave as print than to mutate the logger. + # Cleaner to leave as print than to mutate the logger not to print timestamps. print(result) return self.node_results From 7a5b642890a81f0c1ee9bb9ce6c3885124df8391 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Thu, 19 Jan 2023 19:55:42 -0800 Subject: [PATCH 14/17] Simplify code for allowing our GraphTest cases to pass while also hiding compile stats from dbt ls/list . --- core/dbt/compilation.py | 14 +++++++++----- test/unit/test_graph.py | 7 +++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index eb7d33cae34..6b354fd6a6d 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -1,11 +1,12 @@ -import os -from collections import defaultdict -from typing import List, Dict, Any, Tuple, Optional - +import argparse import networkx as nx # type: ignore +import os import pickle import sqlparse +from collections import defaultdict +from typing import List, Dict, Any, Tuple, Optional + from dbt import flags from dbt.adapters.factory import get_adapter from dbt.clients import jinja @@ -476,7 +477,10 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph self.write_graph_file(linker, manifest) # Do not print these for ListTask's - if not self.config.args.cls == list_task.ListTask: + if ( + self.config.args.__class__ == argparse.Namespace + and not self.config.args.cls == list_task.ListTask + ): print_compile_stats(stats) return Graph(linker.graph) diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index 5534fe21f19..3f330717608 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -59,6 +59,7 @@ def setUp(self): # Create file filesystem searcher self.filesystem_search = patch('dbt.parser.read_files.filesystem_search') + def mock_filesystem_search(project, relative_dirs, extension, ignore_spec): if 'sql' not in extension: return [] @@ -72,6 +73,7 @@ def mock_filesystem_search(project, relative_dirs, extension, ignore_spec): self.hook_patcher = patch.object( dbt.parser.hooks.HookParser, '__new__' ) + def create_hook_patcher(cls, project, manifest, root_project): result = MagicMock(project=project, manifest=manifest, root_project=root_project) result.__iter__.side_effect = lambda: iter([]) @@ -82,7 +84,6 @@ def create_hook_patcher(cls, project, manifest, root_project): # Create the Manifest.state_check patcher @patch('dbt.parser.manifest.ManifestLoader.build_manifest_state_check') def _mock_state_check(self): - config = self.root_project all_projects = self.all_projects return ManifestStateCheck( project_env_vars_hash=FileHash.from_contents(''), @@ -98,6 +99,7 @@ def _mock_state_check(self): # Create the source file patcher self.load_source_file_patcher = patch('dbt.parser.read_files.load_source_file') self.mock_source_file = self.load_source_file_patcher.start() + def mock_load_source_file(path, parse_file_type, project_name, saved_files): for sf in self.mock_models: if sf.path == path: @@ -117,7 +119,6 @@ def _mock_hook_path(self): ) return path - def get_config(self, extra_cfg=None): if extra_cfg is None: extra_cfg = {} @@ -224,8 +225,6 @@ def test__model_materializations(self): config = self.get_config(cfg) manifest = self.load_manifest(config) - compiler = self.get_compiler(config) - linker = compiler.compile(manifest) expected_materialization = { "model_one": "table", From befddb36847fd985deead0870070911998cc164f Mon Sep 17 00:00:00 2001 From: Mila Page Date: Thu, 19 Jan 2023 20:20:51 -0800 Subject: [PATCH 15/17] Simplify structured logging types. --- core/dbt/events/proto_types.py | 161 ++------------------------------- core/dbt/events/types.proto | 128 ++------------------------ core/dbt/events/types.py | 105 +-------------------- core/dbt/task/debug.py | 72 +++++---------- core/dbt/task/list.py | 4 +- tests/unit/test_events.py | 17 +--- 6 files changed, 47 insertions(+), 440 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 0ed5ff93939..0bae4be273f 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -2848,185 +2848,42 @@ class RunResultWarningMessageMsg(betterproto.Message): @dataclass -class DebugEnvironmentDetails(betterproto.Message): +class DebugCmdOut(betterproto.Message): """Z047""" msg: str = betterproto.string_field(1) @dataclass -class DebugEnvironmentDetailsMsg(betterproto.Message): +class DebugCmdOutMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DebugEnvironmentDetails" = betterproto.message_field(2) + data: "DebugCmdOut" = betterproto.message_field(2) @dataclass -class DebugDependenciesDetails(betterproto.Message): +class DebugCmdResult(betterproto.Message): """Z048""" msg: str = betterproto.string_field(1) @dataclass -class DebugDependenciesDetailsMsg(betterproto.Message): +class DebugCmdResultMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DebugDependenciesDetails" = betterproto.message_field(2) + data: "DebugCmdResult" = betterproto.message_field(2) @dataclass -class DebugDependenciesSuccess(betterproto.Message): +class ListCmdOut(betterproto.Message): """Z049""" msg: str = betterproto.string_field(1) @dataclass -class DebugDependenciesSuccessMsg(betterproto.Message): +class ListCmdOutMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DebugDependenciesSuccess" = betterproto.message_field(2) - - -@dataclass -class DebugDependenciesFailure(betterproto.Message): - """Z050""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugDependenciesFailureMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugDependenciesFailure" = betterproto.message_field(2) - - -@dataclass -class DebugConfigurationDetails(betterproto.Message): - """Z051""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugConfigurationDetailsMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugConfigurationDetails" = betterproto.message_field(2) - - -@dataclass -class DebugConfigurationSuccess(betterproto.Message): - """Z052""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugConfigurationSuccessMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugConfigurationSuccess" = betterproto.message_field(2) - - -@dataclass -class DebugConfigurationFailure(betterproto.Message): - """QO53""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugConfigurationFailureMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugConfigurationFailure" = betterproto.message_field(2) - - -@dataclass -class DebugConnectionDetails(betterproto.Message): - """Z054""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugConnectionDetailsMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugConnectionDetails" = betterproto.message_field(2) - - -@dataclass -class DebugConnectionSuccess(betterproto.Message): - """Z055""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugConnectionSuccessMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugConnectionSuccess" = betterproto.message_field(2) - - -@dataclass -class DebugConnectionFailure(betterproto.Message): - """Z056""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugConnectionFailureMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugConnectionFailure" = betterproto.message_field(2) - - -@dataclass -class DebugRunSuccess(betterproto.Message): - """Z057""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugRunSuccessMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugRunSuccess" = betterproto.message_field(2) - - -@dataclass -class DebugRunFailure(betterproto.Message): - """Z058""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugRunFailureMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugRunFailure" = betterproto.message_field(2) - - -@dataclass -class DebugMiscMessages(betterproto.Message): - """Z059""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class DebugMiscMessagesMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DebugMiscMessages" = betterproto.message_field(2) - - -@dataclass -class ListRunDetails(betterproto.Message): - """Z060""" - - msg: str = betterproto.string_field(1) - - -@dataclass -class ListRunDetailsMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ListRunDetails" = betterproto.message_field(2) + data: "ListCmdOut" = betterproto.message_field(2) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index da23375e8ab..fc251042005 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -2259,143 +2259,33 @@ message RunResultWarningMessageMsg { } // Z047 -message DebugEnvironmentDetails { +message DebugCmdOut { string msg = 1; } -message DebugEnvironmentDetailsMsg { +message DebugCmdOutMsg { EventInfo info = 1; - DebugEnvironmentDetails data = 2; + DebugCmdOut data = 2; } // Z048 -message DebugDependenciesDetails { +message DebugCmdResult { string msg = 1; } -message DebugDependenciesDetailsMsg { +message DebugCmdResultMsg { EventInfo info = 1; - DebugDependenciesDetails data = 2; + DebugCmdResult data = 2; } // Z049 -message DebugDependenciesSuccess { +message ListCmdOut { string msg = 1; } -message DebugDependenciesSuccessMsg { +message ListCmdOutMsg { EventInfo info = 1; - DebugDependenciesSuccess data = 2; -} - -// Z050 -message DebugDependenciesFailure { - string msg = 1; -} - -message DebugDependenciesFailureMsg { - EventInfo info = 1; - DebugDependenciesFailure data = 2; -} - -// Z051 -message DebugConfigurationDetails { - string msg = 1; -} - -message DebugConfigurationDetailsMsg { - EventInfo info = 1; - DebugConfigurationDetails data = 2; -} - -// Z052 -message DebugConfigurationSuccess { - string msg = 1; -} - -message DebugConfigurationSuccessMsg { - EventInfo info = 1; - DebugConfigurationSuccess data = 2; -} - -// QO53 -message DebugConfigurationFailure { - string msg = 1; -} - -message DebugConfigurationFailureMsg { - EventInfo info = 1; - DebugConfigurationFailure data = 2; -} - -// Z054 -message DebugConnectionDetails { - string msg = 1; -} - -message DebugConnectionDetailsMsg { - EventInfo info = 1; - DebugConnectionDetails data = 2; -} - -// Z055 -message DebugConnectionSuccess { - string msg = 1; -} - -message DebugConnectionSuccessMsg { - EventInfo info = 1; - DebugConnectionSuccess data = 2; -} - -// Z056 -message DebugConnectionFailure { - string msg = 1; -} - -message DebugConnectionFailureMsg { - EventInfo info = 1; - DebugConnectionFailure data = 2; -} - -// Z057 -message DebugRunSuccess { - string msg = 1; -} - -message DebugRunSuccessMsg { - EventInfo info = 1; - DebugRunSuccess data = 2; -} - -// Z058 -message DebugRunFailure { - string msg = 1; -} - -message DebugRunFailureMsg { - EventInfo info = 1; - DebugRunFailure data = 2; -} - -// Z059 -message DebugMiscMessages { - string msg = 1; -} - -message DebugMiscMessagesMsg { - EventInfo info = 1; - DebugMiscMessages data = 2; -} - -// Z060 -message ListRunDetails { - string msg = 1; -} - -message ListRunDetailsMsg { - EventInfo info = 1; - ListRunDetails data = 2; + ListCmdOut data = 2; } diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index c4dfeeb5c16..e74327be179 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2348,7 +2348,7 @@ def message(self) -> str: @dataclass -class DebugEnvironmentDetails(InfoLevel, pt.DebugEnvironmentDetails): +class DebugCmdOut(InfoLevel, pt.DebugCmdOut): def code(self): return "Z047" @@ -2357,7 +2357,7 @@ def message(self) -> str: @dataclass -class DebugDependenciesDetails(InfoLevel, pt.DebugDependenciesDetails): +class DebugCmdResult(InfoLevel, pt.DebugCmdResult): def code(self): return "Z048" @@ -2366,108 +2366,9 @@ def message(self) -> str: @dataclass -class DebugDependenciesSuccess(InfoLevel, pt.DebugDependenciesSuccess): +class ListCmdOut(InfoLevel, pt.ListCmdOut): def code(self): return "Z049" def message(self) -> str: return self.msg - - -@dataclass -class DebugDependenciesFailure(ErrorLevel, pt.DebugDependenciesFailure): - def code(self): - return "Z050" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConfigurationDetails(InfoLevel, pt.DebugConfigurationDetails): - def code(self): - return "Z051" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConfigurationSuccess(InfoLevel, pt.DebugConfigurationSuccess): - def code(self): - return "Z052" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConfigurationFailure(ErrorLevel, pt.DebugConfigurationFailure): - def code(self): - return "Z053" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConnectionDetails(InfoLevel, pt.DebugConnectionDetails): - def code(self): - return "Z054" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConnectionSuccess(InfoLevel, pt.DebugConnectionSuccess): - def code(self): - return "Z055" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugConnectionFailure(ErrorLevel, pt.DebugConnectionFailure): - def code(self): - return "Z056" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugRunSuccess(InfoLevel, pt.DebugRunSuccess): - def code(self): - return "Z057" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugRunFailure(ErrorLevel, pt.DebugRunFailure): - def code(self): - return "Z058" - - def message(self) -> str: - return self.msg - - -@dataclass -class DebugMiscMessages(WarnLevel, pt.DebugMiscMessages): - def code(self): - return "Z059" - - def message(self) -> str: - return self.msg - - -@dataclass -class ListRunDetails(InfoLevel, pt.ListRunDetails): - def code(self): - return "Z060" - - def message(self) -> str: - return self.msg diff --git a/core/dbt/task/debug.py b/core/dbt/task/debug.py index 26a05489905..7460c08cc84 100644 --- a/core/dbt/task/debug.py +++ b/core/dbt/task/debug.py @@ -7,19 +7,8 @@ from dbt.events.functions import fire_event from dbt.events.types import ( OpenCommand, - DebugEnvironmentDetails, - DebugDependenciesDetails, - DebugDependenciesSuccess, - DebugDependenciesFailure, - DebugConfigurationDetails, - DebugConfigurationSuccess, - DebugConfigurationFailure, - DebugConnectionDetails, - DebugConnectionSuccess, - DebugConnectionFailure, - DebugRunSuccess, - DebugRunFailure, - DebugMiscMessages, + DebugCmdOut, + DebugCmdResult, ) from dbt import flags import dbt.clients.system @@ -114,33 +103,25 @@ def run(self): return not self.any_failure version = get_installed_version().to_version_string(skip_matcher=True) - fire_event(DebugEnvironmentDetails(msg="dbt version: {}".format(version))) - fire_event( - DebugEnvironmentDetails(msg="python version: {}".format(sys.version.split()[0])) - ) - fire_event(DebugEnvironmentDetails(msg="python path: {}".format(sys.executable))) - fire_event(DebugEnvironmentDetails(msg="os info: {}".format(platform.platform()))) - fire_event( - DebugEnvironmentDetails(msg="Using profiles.yml file at {}".format(self.profile_path)) - ) - fire_event( - DebugEnvironmentDetails( - msg="Using dbt_project.yml file at {}".format(self.project_path) - ) - ) + fire_event(DebugCmdOut(msg="dbt version: {}".format(version))) + fire_event(DebugCmdOut(msg="python version: {}".format(sys.version.split()[0]))) + fire_event(DebugCmdOut(msg="python path: {}".format(sys.executable))) + fire_event(DebugCmdOut(msg="os info: {}".format(platform.platform()))) + fire_event(DebugCmdOut(msg="Using profiles.yml file at {}".format(self.profile_path))) + fire_event(DebugCmdOut(msg="Using dbt_project.yml file at {}".format(self.project_path))) self.test_configuration() self.test_dependencies() self.test_connection() if self.any_failure: fire_event( - DebugRunFailure(msg=red(f"{(pluralize(len(self.messages), 'check'))} failed:")) + DebugCmdResult(msg=red(f"{(pluralize(len(self.messages), 'check'))} failed:")) ) else: - fire_event(DebugRunSuccess(msg=green("All checks passed!"))) + fire_event(DebugCmdResult(msg=green("All checks passed!"))) for message in self.messages: - fire_event(DebugMiscMessages(msg=f"{message}\n")) + fire_event(DebugCmdResult(msg=f"{message}\n")) return not self.any_failure @@ -296,39 +277,29 @@ def test_git(self): return green("OK found") def test_dependencies(self): - fire_event(DebugDependenciesDetails(msg="Required dependencies:")) + fire_event(DebugCmdOut(msg="Required dependencies:")) logline_msg = self.test_git() - event_type = DebugDependenciesSuccess if "OK" in logline_msg else DebugDependenciesFailure - fire_event(event_type(msg=f" - git [{logline_msg}]\n")) + fire_event(DebugCmdResult(msg=f" - git [{logline_msg}]\n")) def test_configuration(self): - def get_config_result_type(status): - return DebugConfigurationSuccess if "OK" in status else DebugConfigurationFailure - - fire_event(DebugConfigurationDetails(msg="Configuration:")) + fire_event(DebugCmdOut(msg="Configuration:")) profile_status = self._load_profile() - fire_event( - get_config_result_type(profile_status)(msg=f" profiles.yml file [{profile_status}]") - ) + fire_event(DebugCmdOut(msg=f" profiles.yml file [{profile_status}]")) project_status = self._load_project() - fire_event( - get_config_result_type(project_status)( - msg=f" dbt_project.yml file [{project_status}]" - ) - ) + fire_event(DebugCmdOut(msg=f" dbt_project.yml file [{project_status}]")) # skip profile stuff if we can't find a profile name if self.profile_name is not None: fire_event( - DebugConfigurationDetails( + DebugCmdOut( msg=" profile: {} [{}]\n".format(self.profile_name, self._profile_found()) ) ) fire_event( - DebugConfigurationDetails( + DebugCmdOut( msg=" target: {} [{}]\n".format(self.target_name, self._target_found()) ) ) @@ -393,13 +364,12 @@ def _connection_result(self): def test_connection(self): if not self.profile: return - fire_event(DebugConnectionDetails(msg="Connection:")) + fire_event(DebugCmdOut(msg="Connection:")) for k, v in self.profile.credentials.connection_info(): - fire_event(DebugConnectionDetails(msg=f" {k}: {v}")) + fire_event(DebugCmdOut(msg=f" {k}: {v}")) res = self._connection_result() - event_type = DebugConnectionSuccess if "OK" in res else DebugConnectionFailure - fire_event(event_type(msg=f" Connection test: [{res}]\n")) + fire_event(DebugCmdOut(msg=f" Connection test: [{res}]\n")) @classmethod def validate_connection(cls, target_dict): diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index a2e4ffdf302..411bed3da4c 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -13,7 +13,7 @@ ) from dbt.events.types import ( NoNodesSelected, - ListRunDetails, + ListCmdOut, ) from dbt.exceptions import DbtRuntimeError, DbtInternalError @@ -144,7 +144,7 @@ def output_results(self, results): for result in results: self.node_results.append(result) if dbt.flags.LOG_FORMAT == "json": - fire_event(ListRunDetails(msg=result)) + fire_event(ListCmdOut(msg=result)) else: # Cleaner to leave as print than to mutate the logger not to print timestamps. print(result) diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index dd4560690c6..7fa586cefa9 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -383,20 +383,9 @@ def test_event_codes(self): FlushEventsFailure(), TrackingInitializeFailure(), RunResultWarningMessage(), - DebugEnvironmentDetails(), - DebugDependenciesDetails(), - DebugDependenciesSuccess(), - DebugDependenciesFailure(), - DebugConfigurationDetails(), - DebugConfigurationSuccess(), - DebugConfigurationFailure(), - DebugConnectionDetails(), - DebugConnectionSuccess(), - DebugConnectionFailure(), - DebugRunSuccess(), - DebugRunFailure(), - DebugMiscMessages(), - ListRunDetails(), + DebugCmdOut(), + DebugCmdResult(), + ListCmdOut(), # T - tests ====================== IntegrationTestInfo(), IntegrationTestDebug(), From 96a48cbdb847918b8e3e4c09cc38a78eaa59c837 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Thu, 19 Jan 2023 20:25:25 -0800 Subject: [PATCH 16/17] Fix up boolean logic and simplify via De'Morgan. --- core/dbt/compilation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 6b354fd6a6d..a89f36d9f31 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -477,9 +477,9 @@ def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph self.write_graph_file(linker, manifest) # Do not print these for ListTask's - if ( + if not ( self.config.args.__class__ == argparse.Namespace - and not self.config.args.cls == list_task.ListTask + and self.config.args.cls == list_task.ListTask ): print_compile_stats(stats) From 88626961aa5c6c135931c8b1bac4e403601c2149 Mon Sep 17 00:00:00 2001 From: Mila Page Date: Fri, 20 Jan 2023 11:39:07 -0800 Subject: [PATCH 17/17] Nix unneeded fixture. --- tests/functional/profiles/test_profile_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/profiles/test_profile_dir.py b/tests/functional/profiles/test_profile_dir.py index 536323655db..75a30512dcb 100644 --- a/tests/functional/profiles/test_profile_dir.py +++ b/tests/functional/profiles/test_profile_dir.py @@ -89,7 +89,7 @@ def environ(env): class TestProfiles: - def dbt_debug(self, project, project_dir_cli_arg=None, profiles_dir_cli_arg=None): + def dbt_debug(self, project_dir_cli_arg=None, profiles_dir_cli_arg=None): # begin with no command-line args or user config (from profiles.yml) flags.set_from_args({}, {}) command = ["debug"]