Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

profile work in click #6303

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions core/dbt/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from dbt.events.functions import setup_event_logger
from dbt.profiler import profiler
from dbt.tracking import initialize_from_flags, track_run
from dbt.config.runtime import load_project
from dbt.config.runtime import load_project, load_profile
from dbt.task.deps import DepsTask


Expand Down Expand Up @@ -75,9 +75,11 @@ def cli(ctx, **kwargs):
if flags.RECORD_TIMING_INFO:
ctx.with_resource(profiler(enable=True, outfile=flags.RECORD_TIMING_INFO))

# TODO need profile to exisit
profile = None

# TODO: not all commands need profile, need to figure out how to do it progmatically
profile = load_profile(
flags.PROJECT_DIR, flags.VARS, flags.THREADS, flags.TARGET, flags.PROFILE
)
ctx.obj["profile"] = profile
# project need profile to render because it requires knowing Target
ctx.obj["project"] = load_project(flags.PROJECT_DIR, flags.VERSION_CHECK, profile, flags.VARS)
# Adapter management
Expand Down
7 changes: 3 additions & 4 deletions core/dbt/config/profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -401,9 +401,11 @@ def from_raw_profiles(
@classmethod
def render_from_args(
MichelleArk marked this conversation as resolved.
Show resolved Hide resolved
cls,
args: Any,
renderer: ProfileRenderer,
project_profile_name: Optional[str],
threads_override: Optional[int] = None,
target_override: Optional[str] = None,
profile_name_override: Optional[str] = None,
) -> "Profile":
"""Given the raw profiles as read from disk and the name of the desired
profile if specified, return the profile component of the runtime
Expand All @@ -419,9 +421,6 @@ def render_from_args(
target could not be found.
:returns Profile: The new Profile object.
"""
threads_override = getattr(args, "threads", None)
target_override = getattr(args, "target", None)
profile_name_override = getattr(args, "profile", None)
raw_profiles = read_profile(flags.PROFILES_DIR)
profile_name = cls.pick_profile_name(profile_name_override, project_profile_name)
return cls.from_raw_profiles(
Expand Down
81 changes: 50 additions & 31 deletions core/dbt/config/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,25 @@ def load_project(
return project


def load_profile(
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@MichelleArk I imagine we would use something like this to generate the profile object, we maybe should refactor this code to pass in a loaded project dict directly instead of reading from disk, that way we can reuse that loaded project to also construct project object

project_root: str,
cli_vars: Dict[str, Any],
threads_override: Optional[int] = None,
target_override: Optional[str] = None,
profile_name_override: Optional[str] = None,
) -> Profile:
raw_project = load_raw_project(project_root)
raw_profile_name: str = raw_project.get("profile") # type: ignore
profile_renderer = ProfileRenderer(cli_vars)
profile_name = profile_renderer.render_value(raw_profile_name)
profile = Profile.render_from_args(
profile_renderer, profile_name, threads_override, target_override, profile_name_override
)
# Save env_vars encountered in rendering for partial parsing
profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
return profile


def _project_quoting_dict(proj: Project, profile: Profile) -> Dict[ComponentName, bool]:
src: Dict[str, Any] = profile.credentials.translate_aliases(proj.quoting)
result: Dict[ComponentName, bool] = {}
Expand All @@ -66,6 +85,21 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
cli_vars: Dict[str, Any]
dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None

@classmethod
def get_profile(
cls,
project_root: str,
cli_vars: Dict[str, Any],
args: Any,
) -> Profile:
return load_profile(
project_root,
cli_vars,
getattr(args, "threads", None),
getattr(args, "target", None),
getattr(args, "profile", None),
)

def __post_init__(self):
self.validate()

Expand Down Expand Up @@ -196,39 +230,17 @@ def validate(self):
except ValidationError as e:
raise DbtProjectError(validator_error_message(e)) from e

@classmethod
def _get_rendered_profile(
cls,
args: Any,
profile_renderer: ProfileRenderer,
profile_name: Optional[str],
) -> Profile:

return Profile.render_from_args(args, profile_renderer, profile_name)

@classmethod
def get_profile(
cls: Type["RuntimeConfig"], args: Any, cli_vars: Dict[str, Any], raw_profile_name: str
) -> Profile:
# build the profile using the base renderer and the one fact we know
# Note: only the named profile section is rendered. The rest of the
# profile is ignored.
profile_renderer = ProfileRenderer(cli_vars)
profile_name = profile_renderer.render_value(raw_profile_name)
profile = cls._get_rendered_profile(args, profile_renderer, profile_name)
MichelleArk marked this conversation as resolved.
Show resolved Hide resolved
# Save env_vars encountered in rendering for partial parsing
profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
return profile

@classmethod
def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
# profile_name from the project
project_root = args.project_dir if args.project_dir else os.getcwd()
raw_project = load_raw_project(project_root)
raw_profile_name: str = raw_project.get("profile") # type: ignore
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))

profile = cls.get_profile(args, cli_vars, raw_profile_name)
profile = cls.get_profile(
project_root,
cli_vars,
args,
)

project = load_project(project_root, bool(flags.VERSION_CHECK), profile, cli_vars)

Expand Down Expand Up @@ -582,18 +594,25 @@ def from_parts(
)

@classmethod
def _get_rendered_profile(
def get_profile(
cls,
project_root: str,
cli_vars: Dict[str, Any],
args: Any,
profile_renderer: ProfileRenderer,
profile_name: Optional[str],
) -> Profile:
"""
Moving all logic regarding constructing a complete UnsetProfile to this function
This way we can have a clean load_profile function to call by the new CLI and remove
all logic for UnsetProfile once we migrate to new click CLI
"""

profile = UnsetProfile()
# The profile (for warehouse connection) is not needed, but we want
# to get the UserConfig, which is also in profiles.yml
user_config = read_user_config(flags.PROFILES_DIR)
user_config = read_user_config(project_root)
profile.user_config = user_config
profile_renderer = ProfileRenderer(cli_vars)
profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
return profile

@classmethod
Expand Down
8 changes: 7 additions & 1 deletion core/dbt/config/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,13 @@ def get_project_config(
flags.set_from_args(args, user_config)
if cli_vars is None:
cli_vars = {}
profile = Profile.render_from_args(args, ProfileRenderer(cli_vars), profile_name)
profile = Profile.render_from_args(
ProfileRenderer(cli_vars),
profile_name,
getattr(args, "threads", None),
getattr(args, "target", None),
getattr(args, "profile", None),
)
# Generate a project
project = Project.from_project_root(
project_path,
Expand Down
8 changes: 7 additions & 1 deletion core/dbt/task/debug.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,13 @@ def _load_profile(self):
renderer = ProfileRenderer(self.cli_vars)
for profile_name in profile_names:
try:
profile: Profile = Profile.render_from_args(self.args, renderer, profile_name)
profile: Profile = Profile.render_from_args(
renderer,
profile_name,
getattr(self.args, "threads", None),
getattr(self.args, "target", None),
getattr(self.args, "profile", None),
)
except dbt.exceptions.DbtConfigError as exc:
profile_errors.append(str(exc))
else:
Expand Down