From 32c559838d692c572a6a33446c5d71b63b88d257 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 23 Jun 2020 10:14:24 -0600 Subject: [PATCH] Try to make imports a little more sane, ordering-wise consolidate dbt.ui, move non-rpc node_runners into their tasks move parse_cli_vars into config.utils get rid of logger/exceptions requirements in dbt.utils --- core/dbt/adapters/base/connections.py | 8 +- core/dbt/adapters/base/impl.py | 14 +- core/dbt/adapters/sql/connections.py | 7 +- core/dbt/adapters/sql/impl.py | 1 - core/dbt/clients/jinja.py | 6 +- core/dbt/compilation.py | 6 +- core/dbt/config/runtime.py | 6 +- core/dbt/config/utils.py | 23 + core/dbt/contracts/graph/model_config.py | 2 +- core/dbt/contracts/graph/parsed.py | 6 +- core/dbt/contracts/project.py | 6 +- core/dbt/deprecations.py | 6 +- core/dbt/deps/git.py | 4 +- core/dbt/exceptions.py | 8 +- core/dbt/logger.py | 17 +- core/dbt/main.py | 6 +- core/dbt/node_runners.py | 662 ------------------ core/dbt/parser/base.py | 6 +- core/dbt/parser/manifest.py | 65 +- core/dbt/parser/results.py | 4 +- core/dbt/parser/sources.py | 4 +- core/dbt/rpc/gc.py | 1 - core/dbt/rpc/node_runners.py | 2 +- core/dbt/rpc/task_handler.py | 6 +- core/dbt/rpc/task_handler_protocol.py | 1 - core/dbt/rpc/task_manager.py | 7 +- core/dbt/task/base.py | 296 +++++++- core/dbt/task/compile.py | 26 +- core/dbt/task/debug.py | 18 +- core/dbt/task/freshness.py | 101 ++- core/dbt/task/generate.py | 15 +- core/dbt/{ui => task}/printer.py | 144 +--- core/dbt/task/rpc/cli.py | 2 +- core/dbt/task/run.py | 194 ++++- core/dbt/task/run_operation.py | 13 +- core/dbt/task/runnable.py | 31 +- core/dbt/task/seed.py | 35 +- core/dbt/task/snapshot.py | 16 +- core/dbt/task/test.py | 91 ++- core/dbt/ui.py | 76 ++ core/dbt/ui/__init__.py | 0 core/dbt/ui/colors.py | 10 - core/dbt/utils.py | 75 -- .../bigquery/dbt/adapters/bigquery/impl.py | 9 +- test/unit/test_main.py | 2 +- test/unit/utils.py | 4 +- 46 files changed, 1033 insertions(+), 1009 deletions(-) create mode 100644 core/dbt/config/utils.py delete mode 100644 core/dbt/node_runners.py rename core/dbt/{ui => task}/printer.py (71%) create mode 100644 core/dbt/ui.py delete mode 100644 core/dbt/ui/__init__.py delete mode 100644 core/dbt/ui/colors.py diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 07088dc0083..e39d3681403 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -10,7 +10,6 @@ import agate import dbt.exceptions -import dbt.flags from dbt.contracts.connection import ( Connection, Identifier, ConnectionState, AdapterRequiredConfig, LazyHandle ) @@ -19,6 +18,7 @@ MacroQueryStringSetter, ) from dbt.logger import GLOBAL_LOGGER as logger +from dbt import flags class BaseConnectionManager(metaclass=abc.ABCMeta): @@ -39,7 +39,7 @@ class BaseConnectionManager(metaclass=abc.ABCMeta): def __init__(self, profile: AdapterRequiredConfig): self.profile = profile self.thread_connections: Dict[Hashable, Connection] = {} - self.lock: RLock = dbt.flags.MP_CONTEXT.RLock() + self.lock: RLock = flags.MP_CONTEXT.RLock() self.query_header: Optional[MacroQueryStringSetter] = None def set_query_header(self, manifest: Manifest) -> None: @@ -235,7 +235,7 @@ def _close_handle(cls, connection: Connection) -> None: @classmethod def _rollback(cls, connection: Connection) -> None: """Roll back the given connection.""" - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In _rollback, got {connection} - not a Connection!' @@ -253,7 +253,7 @@ def _rollback(cls, connection: Connection) -> None: @classmethod def close(cls, connection: Connection) -> Connection: - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In close, got {connection} - not a Connection!' diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 6af38b5264b..ab6064bb90b 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -17,7 +17,7 @@ get_relation_returned_multiple_results, InternalException, NotImplementedException, RuntimeException, ) -import dbt.flags +from dbt import flags from dbt import deprecations from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows @@ -267,7 +267,7 @@ def load_internal_manifest(self) -> Manifest: def _schema_is_cached(self, database: Optional[str], schema: str) -> bool: """Check if the schema is cached, and by default logs if it is not.""" - if dbt.flags.USE_CACHE is False: + if flags.USE_CACHE is False: return False elif (database, schema) not in self.cache: logger.debug( @@ -323,7 +323,7 @@ def _relations_cache_for_schemas(self, manifest: Manifest) -> None: """Populate the relations cache for the given schemas. Returns an iterable of the schemas populated, as strings. """ - if not dbt.flags.USE_CACHE: + if not flags.USE_CACHE: return cache_schemas = self._get_cache_schemas(manifest) @@ -352,7 +352,7 @@ def set_relations_cache( """Run a query that gets a populated cache of the relations in the database and set the cache on this adapter. """ - if not dbt.flags.USE_CACHE: + if not flags.USE_CACHE: return with self.cache.lock: @@ -368,7 +368,7 @@ def cache_added(self, relation: Optional[BaseRelation]) -> str: raise_compiler_error( 'Attempted to cache a null relation for {}'.format(name) ) - if dbt.flags.USE_CACHE: + if flags.USE_CACHE: self.cache.add(relation) # so jinja doesn't render things return '' @@ -383,7 +383,7 @@ def cache_dropped(self, relation: Optional[BaseRelation]) -> str: raise_compiler_error( 'Attempted to drop a null relation for {}'.format(name) ) - if dbt.flags.USE_CACHE: + if flags.USE_CACHE: self.cache.drop(relation) return '' @@ -405,7 +405,7 @@ def cache_renamed( .format(src_name, dst_name, name) ) - if dbt.flags.USE_CACHE: + if flags.USE_CACHE: self.cache.rename(from_relation, to_relation) return '' diff --git a/core/dbt/adapters/sql/connections.py b/core/dbt/adapters/sql/connections.py index 3fe1dad2ec7..91ebe4e0d85 100644 --- a/core/dbt/adapters/sql/connections.py +++ b/core/dbt/adapters/sql/connections.py @@ -6,9 +6,10 @@ import dbt.clients.agate_helper import dbt.exceptions -from dbt.contracts.connection import Connection from dbt.adapters.base import BaseConnectionManager +from dbt.contracts.connection import Connection from dbt.logger import GLOBAL_LOGGER as logger +from dbt import flags class SQLConnectionManager(BaseConnectionManager): @@ -133,7 +134,7 @@ def add_commit_query(self): def begin(self): connection = self.get_thread_connection() - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In begin, got {connection} - not a Connection!' @@ -151,7 +152,7 @@ def begin(self): def commit(self): connection = self.get_thread_connection() - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In commit, got {connection} - not a Connection!' diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index 84268b34d16..93b7aae0906 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -4,7 +4,6 @@ import dbt.clients.agate_helper from dbt.contracts.connection import Connection import dbt.exceptions -import dbt.flags from dbt.adapters.base import BaseAdapter, available from dbt.adapters.sql import SQLConnectionManager from dbt.logger import GLOBAL_LOGGER as logger diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 6138e8902c8..07a344338b7 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -30,7 +30,7 @@ InternalException, raise_compiler_error, CompilationException, invalid_materialization_argument, MacroReturn ) -from dbt.flags import MACRO_DEBUGGING +from dbt import flags from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -93,8 +93,8 @@ def _compile(self, source, filename): If the value is 'write', also write the files to disk. WARNING: This can write a ton of data if you aren't careful. """ - if filename == '