From 238ce4da73d1e618409e647c915599a2866c3355 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Mon, 9 Jan 2023 15:51:03 +0000 Subject: [PATCH] Move environment variable resolution to load_and_merge_dir_config Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index b79c44d82e..0896c641a6 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -4,7 +4,7 @@ import logging from glob import iglob from pathlib import Path -from typing import Any, Dict, Iterable, List, Set # noqa +from typing import Any, Dict, Iterable, List, Optional, Set # noqa from omegaconf import OmegaConf from omegaconf.resolvers import oc @@ -144,15 +144,21 @@ def __getitem__(self, key) -> Dict[str, Any]: ) patterns = [*self.config_patterns[key]] + read_environment_variables = key == "credentials" + # Load base env config base_path = str(Path(self.conf_source) / self.base_env) - base_config = self.load_and_merge_dir_config(base_path, patterns) + base_config = self.load_and_merge_dir_config( + base_path, patterns, read_environment_variables + ) config = base_config # Load chosen env config run_env = self.env or self.default_run_env env_path = str(Path(self.conf_source) / run_env) - env_config = self.load_and_merge_dir_config(env_path, patterns) + env_config = self.load_and_merge_dir_config( + env_path, patterns, read_environment_variables + ) # Destructively merge the two env dirs. The chosen env will override base. common_keys = config.keys() & env_config.keys() @@ -166,9 +172,6 @@ def __getitem__(self, key) -> Dict[str, Any]: config.update(env_config) - if key == "credentials": - self._resolve_environment_variables(env_config) - if not config: raise MissingConfigException( f"No files of YAML or JSON format found in {base_path} or {env_path} matching" @@ -182,7 +185,12 @@ def __repr__(self): # pragma: no cover f"config_patterns={self.config_patterns})" ) - def load_and_merge_dir_config(self, conf_path: str, patterns: Iterable[str]): + def load_and_merge_dir_config( + self, + conf_path: str, + patterns: Iterable[str], + read_environment_variables: Optional[bool] = False, + ) -> Dict[str, Any]: """Recursively load and merge all configuration files in a directory using OmegaConf, which satisfy a given list of glob patterns from a specific path. @@ -220,6 +228,8 @@ def load_and_merge_dir_config(self, conf_path: str, patterns: Iterable[str]): for config_filepath in config_files_filtered: try: config = OmegaConf.load(config_filepath) + if read_environment_variables: + self._resolve_environment_variables(config) config_per_file[config_filepath] = config except (ParserError, ScannerError) as exc: line = exc.problem_mark.line # type: ignore