From 20fc619198663a43c6b65d968b8a266c73e9f4e1 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Tue, 3 Dec 2024 16:36:18 -0600 Subject: [PATCH 1/4] improve global and runtime param resolver --- kedro/config/omegaconf_config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 8d82ebf360..2a109e6ba7 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -126,7 +126,8 @@ def __init__( # noqa: PLR0913 self.base_env = base_env or "" self.default_run_env = default_run_env or "" self.merge_strategy = merge_strategy or {} - + self._globals_oc = None + self._runtime_params_oc = None self.config_patterns = { "catalog": ["catalog*", "catalog*/**", "**/catalog*"], "parameters": ["parameters*", "parameters*/**", "**/parameters*"], @@ -436,9 +437,12 @@ def _get_globals_value(self, variable: str, default_value: Any = _NO_VALUE) -> A raise InterpolationResolutionError( "Keys starting with '_' are not supported for globals." ) - globals_oc = OmegaConf.create(self._globals) + + if not self._globals_oc: + self._globals_oc = OmegaConf.create(self._globals) + interpolated_value = OmegaConf.select( - globals_oc, variable, default=default_value + self._globals_oc, variable, default=default_value ) if interpolated_value != _NO_VALUE: return interpolated_value @@ -449,9 +453,11 @@ def _get_globals_value(self, variable: str, default_value: Any = _NO_VALUE) -> A def _get_runtime_value(self, variable: str, default_value: Any = _NO_VALUE) -> Any: """Return the runtime params values to the resolver""" - runtime_oc = OmegaConf.create(self.runtime_params) + if not self._runtime_params_oc: + self._runtime_params_oc = OmegaConf.create(self.runtime_params) + interpolated_value = OmegaConf.select( - runtime_oc, variable, default=default_value + self._runtime_params_oc, variable, default=default_value ) if interpolated_value != _NO_VALUE: return interpolated_value From c256a5fec5bd85badab63e69eadc78aaf11fefb5 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Sun, 22 Dec 2024 23:19:14 -0600 Subject: [PATCH 2/4] testing with slack suggestions --- kedro/config/omegaconf_config.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 2a109e6ba7..73758f54b6 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -347,12 +347,11 @@ def load_and_merge_dir_config( OmegaConf.merge(*aggregate_config, self.runtime_params), resolve=True ) + merged_config_container = OmegaConf.to_container( + OmegaConf.merge(*aggregate_config), resolve=True + ) return { - k: v - for k, v in OmegaConf.to_container( - OmegaConf.merge(*aggregate_config), resolve=True - ).items() - if not k.startswith("_") + k: v for k, v in merged_config_container.items() if not k.startswith("_") } @staticmethod From 2d7fdff33467decf552027b3b4fec81f5e59c48b Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Mon, 23 Dec 2024 12:17:48 -0600 Subject: [PATCH 3/4] fix type lint issue --- kedro/config/omegaconf_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 73758f54b6..3f40624a4b 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -126,8 +126,8 @@ def __init__( # noqa: PLR0913 self.base_env = base_env or "" self.default_run_env = default_run_env or "" self.merge_strategy = merge_strategy or {} - self._globals_oc = None - self._runtime_params_oc = None + self._globals_oc: DictConfig | None = None + self._runtime_params_oc: DictConfig | None = None self.config_patterns = { "catalog": ["catalog*", "catalog*/**", "**/catalog*"], "parameters": ["parameters*", "parameters*/**", "**/parameters*"], From 4eb7fac9cd9ca5e1e050048bb955b9df80f1e150 Mon Sep 17 00:00:00 2001 From: ravi_kumar_pilla Date: Fri, 3 Jan 2025 11:07:36 -0600 Subject: [PATCH 4/4] update release note Signed-off-by: ravi_kumar_pilla --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 7af26f6ca4..c19418f0ca 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,6 +2,7 @@ ## Major features and improvements * Implemented `KedroDataCatalog.to_config()` method that converts the catalog instance into a configuration format suitable for serialization. +* Improve OmegaConfigLoader performance ## Bug fixes and other changes * Added validation to ensure dataset versions consistency across catalog.