From d0cbcb8b18bf6e43c03f1ac789695687efc0073e Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Fri, 6 Jan 2023 16:16:38 +0000 Subject: [PATCH 01/14] Add failing test Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index aebb0e3221..b307a393da 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -2,6 +2,7 @@ import configparser import json import re +import os from pathlib import Path from typing import Dict @@ -86,6 +87,12 @@ def proj_catalog_nested(tmp_path): _write_yaml(path, {"nested": {"type": "MemoryDataSet"}}) +@pytest.fixture +def credentials_env_variables(tmp_path): + path = tmp_path / _DEFAULT_RUN_ENV / "credentials.yml" + _write_yaml(path, {"user": {"name": "${oc.env:TEST_USERNAME}", "key": "${oc.env:TEST_KEY}"}}) + + use_config_dir = pytest.mark.usefixtures("create_config_dir") use_proj_catalog = pytest.mark.usefixtures("proj_catalog") @@ -421,3 +428,12 @@ def test_bypass_catalog_config_loading(self, tmp_path): conf["catalog"] = {"catalog_config": "something_new"} assert conf["catalog"] == {"catalog_config": "something_new"} + + @use_config_dir + def test_load_credentials_from_env_variables(self, tmp_path, credentials_env_variables): + """Load credentials from environment variables""" + conf = OmegaConfLoader(str(tmp_path)) + os.environ["TEST_USERNAME"] = "test_user" + os.environ["TEST_KEY"] = "test_key" + assert conf["credentials"]["user"]["name"] == "test_user" + assert conf["credentials"]["user"]["key"] == "test_key" From c0d784f2a9f0313c560a042d2cfb00d3dff0c3b9 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Fri, 6 Jan 2023 16:59:15 +0000 Subject: [PATCH 02/14] Add in-place environment variable resolution Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index ce4efe386f..2e7e40090c 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Iterable, List, Set # noqa from omegaconf import OmegaConf +from omegaconf.resolvers import oc from yaml.parser import ParserError from yaml.scanner import ScannerError @@ -165,6 +166,9 @@ def __getitem__(self, key) -> Dict[str, Any]: config.update(env_config) + if key == "credentials": + self._resolve_environment_variables(env_config) + if not config: raise MissingConfigException( f"No files of YAML or JSON format found in {base_path} or {env_path} matching" @@ -266,6 +270,19 @@ def _check_duplicates(seen_files_to_keys: Dict[Path, Set[Any]]): dup_str = "\n".join(duplicates) raise ValueError(f"{dup_str}") + @staticmethod + def _resolve_environment_variables(config: Dict[str, Any]) -> None: + """Use the oc.env resolver to read environment variables and replace them + in-place, without leaving the resolver on. + + Arguments: + config {Dict[str, Any]} -- The configuration dictionary to resolve. + """ + if not OmegaConf.has_resolver("oc.env"): + OmegaConf.register_new_resolver("oc.env", oc.env) + OmegaConf.resolve(config) + OmegaConf.clear_resolver("oc.env") + @staticmethod def _clear_omegaconf_resolvers(): """Clear the built-in OmegaConf resolvers.""" From 74da5fcd99e225ffa183e1a259a8ba762f4ef0b3 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Fri, 6 Jan 2023 17:10:46 +0000 Subject: [PATCH 03/14] Add test to ensure oc.env resolver is cleared after reading credentials Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index b307a393da..c3e3ea1f02 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -1,13 +1,14 @@ # pylint: disable=expression-not-assigned, pointless-statement import configparser import json -import re import os +import re from pathlib import Path from typing import Dict import pytest import yaml +from omegaconf import OmegaConf from yaml.parser import ParserError from kedro.config import MissingConfigException, OmegaConfLoader @@ -90,7 +91,9 @@ def proj_catalog_nested(tmp_path): @pytest.fixture def credentials_env_variables(tmp_path): path = tmp_path / _DEFAULT_RUN_ENV / "credentials.yml" - _write_yaml(path, {"user": {"name": "${oc.env:TEST_USERNAME}", "key": "${oc.env:TEST_KEY}"}}) + _write_yaml( + path, {"user": {"name": "${oc.env:TEST_USERNAME}", "key": "${oc.env:TEST_KEY}"}} + ) use_config_dir = pytest.mark.usefixtures("create_config_dir") @@ -430,10 +433,22 @@ def test_bypass_catalog_config_loading(self, tmp_path): assert conf["catalog"] == {"catalog_config": "something_new"} @use_config_dir - def test_load_credentials_from_env_variables(self, tmp_path, credentials_env_variables): + def test_load_credentials_from_env_variables( + self, tmp_path, credentials_env_variables + ): """Load credentials from environment variables""" conf = OmegaConfLoader(str(tmp_path)) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" assert conf["credentials"]["user"]["key"] == "test_key" + + @use_config_dir + def test_env_resolver_is_cleared_after_credentials_loading( + self, tmp_path, credentials_env_variables + ): + conf = OmegaConfLoader(str(tmp_path)) + os.environ["TEST_USERNAME"] = "test_user" + os.environ["TEST_KEY"] = "test_key" + assert conf["credentials"]["user"]["name"] == "test_user" + assert not OmegaConf.has_resolver("oc.env") From 3149eebd018fdde59f030544af94d05aa6c27266 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Fri, 6 Jan 2023 17:11:06 +0000 Subject: [PATCH 04/14] Refactor docstring on _resolve_environment_variables Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 2e7e40090c..76ac767cb5 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -272,8 +272,8 @@ def _check_duplicates(seen_files_to_keys: Dict[Path, Set[Any]]): @staticmethod def _resolve_environment_variables(config: Dict[str, Any]) -> None: - """Use the oc.env resolver to read environment variables and replace them - in-place, without leaving the resolver on. + """Use the ``oc.env`` resolver to read environment variables and replace + them in-place, clearing the resolver after the operation is complete. Arguments: config {Dict[str, Any]} -- The configuration dictionary to resolve. From d07cb3bfefbe035584212eada9d648eecf0fcedc Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Mon, 9 Jan 2023 11:24:05 +0000 Subject: [PATCH 05/14] Move credentials.yml creation to fixture Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index c3e3ea1f02..6c013a2db4 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -89,7 +89,7 @@ def proj_catalog_nested(tmp_path): @pytest.fixture -def credentials_env_variables(tmp_path): +def create_credentials_yml(tmp_path): path = tmp_path / _DEFAULT_RUN_ENV / "credentials.yml" _write_yaml( path, {"user": {"name": "${oc.env:TEST_USERNAME}", "key": "${oc.env:TEST_KEY}"}} @@ -98,6 +98,7 @@ def credentials_env_variables(tmp_path): use_config_dir = pytest.mark.usefixtures("create_config_dir") use_proj_catalog = pytest.mark.usefixtures("proj_catalog") +use_credentials_yml = pytest.mark.usefixtures("create_credentials_yml") class TestOmegaConfLoader: @@ -433,9 +434,8 @@ def test_bypass_catalog_config_loading(self, tmp_path): assert conf["catalog"] == {"catalog_config": "something_new"} @use_config_dir - def test_load_credentials_from_env_variables( - self, tmp_path, credentials_env_variables - ): + @use_credentials_yml + def test_load_credentials_from_env_variables(self, tmp_path): """Load credentials from environment variables""" conf = OmegaConfLoader(str(tmp_path)) os.environ["TEST_USERNAME"] = "test_user" @@ -444,9 +444,9 @@ def test_load_credentials_from_env_variables( assert conf["credentials"]["user"]["key"] == "test_key" @use_config_dir - def test_env_resolver_is_cleared_after_credentials_loading( - self, tmp_path, credentials_env_variables - ): + @use_credentials_yml + def test_env_resolver_is_cleared_after_credentials_loading(self, tmp_path): + """Check that the oc.env resolver is cleared after loading credentials""" conf = OmegaConfLoader(str(tmp_path)) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" From 26aa77003640a9adcbd9826d9694125229dcd3ae Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Mon, 9 Jan 2023 11:57:58 +0000 Subject: [PATCH 06/14] Modify _resolve_environment_variables to only clear the oc.env resolver if it was not registered Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 76ac767cb5..b79c44d82e 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -273,15 +273,19 @@ def _check_duplicates(seen_files_to_keys: Dict[Path, Set[Any]]): @staticmethod def _resolve_environment_variables(config: Dict[str, Any]) -> None: """Use the ``oc.env`` resolver to read environment variables and replace - them in-place, clearing the resolver after the operation is complete. + them in-place, clearing the resolver after the operation is complete if + it was not registered beforehand. Arguments: config {Dict[str, Any]} -- The configuration dictionary to resolve. """ + clear_resolver = False if not OmegaConf.has_resolver("oc.env"): OmegaConf.register_new_resolver("oc.env", oc.env) + clear_resolver = True OmegaConf.resolve(config) - OmegaConf.clear_resolver("oc.env") + if clear_resolver: + OmegaConf.clear_resolver("oc.env") @staticmethod def _clear_omegaconf_resolvers(): From 238ce4da73d1e618409e647c915599a2866c3355 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Mon, 9 Jan 2023 15:51:03 +0000 Subject: [PATCH 07/14] Move environment variable resolution to load_and_merge_dir_config Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index b79c44d82e..0896c641a6 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -4,7 +4,7 @@ import logging from glob import iglob from pathlib import Path -from typing import Any, Dict, Iterable, List, Set # noqa +from typing import Any, Dict, Iterable, List, Optional, Set # noqa from omegaconf import OmegaConf from omegaconf.resolvers import oc @@ -144,15 +144,21 @@ def __getitem__(self, key) -> Dict[str, Any]: ) patterns = [*self.config_patterns[key]] + read_environment_variables = key == "credentials" + # Load base env config base_path = str(Path(self.conf_source) / self.base_env) - base_config = self.load_and_merge_dir_config(base_path, patterns) + base_config = self.load_and_merge_dir_config( + base_path, patterns, read_environment_variables + ) config = base_config # Load chosen env config run_env = self.env or self.default_run_env env_path = str(Path(self.conf_source) / run_env) - env_config = self.load_and_merge_dir_config(env_path, patterns) + env_config = self.load_and_merge_dir_config( + env_path, patterns, read_environment_variables + ) # Destructively merge the two env dirs. The chosen env will override base. common_keys = config.keys() & env_config.keys() @@ -166,9 +172,6 @@ def __getitem__(self, key) -> Dict[str, Any]: config.update(env_config) - if key == "credentials": - self._resolve_environment_variables(env_config) - if not config: raise MissingConfigException( f"No files of YAML or JSON format found in {base_path} or {env_path} matching" @@ -182,7 +185,12 @@ def __repr__(self): # pragma: no cover f"config_patterns={self.config_patterns})" ) - def load_and_merge_dir_config(self, conf_path: str, patterns: Iterable[str]): + def load_and_merge_dir_config( + self, + conf_path: str, + patterns: Iterable[str], + read_environment_variables: Optional[bool] = False, + ) -> Dict[str, Any]: """Recursively load and merge all configuration files in a directory using OmegaConf, which satisfy a given list of glob patterns from a specific path. @@ -220,6 +228,8 @@ def load_and_merge_dir_config(self, conf_path: str, patterns: Iterable[str]): for config_filepath in config_files_filtered: try: config = OmegaConf.load(config_filepath) + if read_environment_variables: + self._resolve_environment_variables(config) config_per_file[config_filepath] = config except (ParserError, ScannerError) as exc: line = exc.problem_mark.line # type: ignore From fc7f81e59c431b790908b6ae5e4dc64399eaf243 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Mon, 9 Jan 2023 15:54:01 +0000 Subject: [PATCH 08/14] Add read_environment_variables to load_and_merge_dir_config docstring Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 0896c641a6..082baf106a 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -197,6 +197,7 @@ def load_and_merge_dir_config( Args: conf_path: Path to configuration directory. patterns: List of glob patterns to match the filenames against. + read_environment_variables: Whether to resolve environment variables. Raises: MissingConfigException: If configuration path doesn't exist or isn't valid. From e8fac8e615414eeb3ad19c9496a0a3bcf1517298 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Tue, 10 Jan 2023 15:02:52 +0000 Subject: [PATCH 09/14] Add test for env resolver not being used when config key is not 'credentials' Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 6c013a2db4..9efae3a80f 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -8,7 +8,7 @@ import pytest import yaml -from omegaconf import OmegaConf +from omegaconf import errors, OmegaConf from yaml.parser import ParserError from kedro.config import MissingConfigException, OmegaConfLoader @@ -89,7 +89,13 @@ def proj_catalog_nested(tmp_path): @pytest.fixture -def create_credentials_yml(tmp_path): +def proj_catalog_env_variable(tmp_path): + path = tmp_path / _BASE_ENV / "catalog" / "dir" / "nested.yml" + _write_yaml(path, {"test": {"file_path": "${oc.env:TEST_FILE_PATH}"}}) + + +@pytest.fixture +def proj_credentials_env_variable(tmp_path): path = tmp_path / _DEFAULT_RUN_ENV / "credentials.yml" _write_yaml( path, {"user": {"name": "${oc.env:TEST_USERNAME}", "key": "${oc.env:TEST_KEY}"}} @@ -98,7 +104,8 @@ def create_credentials_yml(tmp_path): use_config_dir = pytest.mark.usefixtures("create_config_dir") use_proj_catalog = pytest.mark.usefixtures("proj_catalog") -use_credentials_yml = pytest.mark.usefixtures("create_credentials_yml") +use_credentials_env_variable_yml = pytest.mark.usefixtures("proj_credentials_env_variable") +use_catalog_env_variable_yml = pytest.mark.usefixtures("proj_catalog_env_variable") class TestOmegaConfLoader: @@ -434,7 +441,7 @@ def test_bypass_catalog_config_loading(self, tmp_path): assert conf["catalog"] == {"catalog_config": "something_new"} @use_config_dir - @use_credentials_yml + @use_credentials_env_variable_yml def test_load_credentials_from_env_variables(self, tmp_path): """Load credentials from environment variables""" conf = OmegaConfLoader(str(tmp_path)) @@ -444,7 +451,7 @@ def test_load_credentials_from_env_variables(self, tmp_path): assert conf["credentials"]["user"]["key"] == "test_key" @use_config_dir - @use_credentials_yml + @use_credentials_env_variable_yml def test_env_resolver_is_cleared_after_credentials_loading(self, tmp_path): """Check that the oc.env resolver is cleared after loading credentials""" conf = OmegaConfLoader(str(tmp_path)) @@ -452,3 +459,12 @@ def test_env_resolver_is_cleared_after_credentials_loading(self, tmp_path): os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" assert not OmegaConf.has_resolver("oc.env") + + @use_config_dir + @use_catalog_env_variable_yml + def test_env_resolver_not_used_for_catalog(self, tmp_path): + """Check that the oc.env resolver is not used for catalog loading""" + conf = OmegaConfLoader(str(tmp_path)) + os.environ["TEST_DATASET"] = "test_dataset" + with pytest.raises(errors.UnsupportedInterpolationType): + conf["catalog"]["test"]["file_path"] From c3ca5167aa9f01c120bf080af5e76e640fda8176 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Tue, 10 Jan 2023 15:17:44 +0000 Subject: [PATCH 10/14] Lint Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 9efae3a80f..9520afb632 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -8,7 +8,7 @@ import pytest import yaml -from omegaconf import errors, OmegaConf +from omegaconf import OmegaConf, errors from yaml.parser import ParserError from kedro.config import MissingConfigException, OmegaConfLoader @@ -104,7 +104,9 @@ def proj_credentials_env_variable(tmp_path): use_config_dir = pytest.mark.usefixtures("create_config_dir") use_proj_catalog = pytest.mark.usefixtures("proj_catalog") -use_credentials_env_variable_yml = pytest.mark.usefixtures("proj_credentials_env_variable") +use_credentials_env_variable_yml = pytest.mark.usefixtures( + "proj_credentials_env_variable" +) use_catalog_env_variable_yml = pytest.mark.usefixtures("proj_catalog_env_variable") From ca83b47f4321f55a19c3a44ce24048a7fe269a36 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Tue, 10 Jan 2023 15:17:51 +0000 Subject: [PATCH 11/14] Add release note Signed-off-by: Jannic Holzer --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index e72d8c00d1..043a9bbd6c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,7 @@ * Added support for generator functions as nodes, i.e. using `yield` instead of return. * Enable chunk-wise processing in nodes with generator functions. * Save node outputs after every `yield` before proceeding with next chunk. +* Added support for loading credentials from environment variables using OmegaConfLoader. ## Bug fixes and other changes * Commas surrounded by square brackets (only possible for nodes with default names) will no longer split the arguments to `kedro run` options which take a list of nodes as inputs (`--from-nodes` and `--to-nodes`). From 6e3c7c821d490b19a86c4dc854fdb534dc490fec Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Tue, 10 Jan 2023 15:37:07 +0000 Subject: [PATCH 12/14] Refactor _resolve_environment_variables to remove unnecessary logic control flow variable Signed-off-by: Jannic Holzer --- kedro/config/omegaconf_config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index a335a8f9f0..fe0d43a746 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -290,13 +290,12 @@ def _resolve_environment_variables(config: Dict[str, Any]) -> None: Arguments: config {Dict[str, Any]} -- The configuration dictionary to resolve. """ - clear_resolver = False if not OmegaConf.has_resolver("oc.env"): OmegaConf.register_new_resolver("oc.env", oc.env) - clear_resolver = True - OmegaConf.resolve(config) - if clear_resolver: + OmegaConf.resolve(config) OmegaConf.clear_resolver("oc.env") + else: + OmegaConf.resolve(config) @staticmethod def _clear_omegaconf_resolvers(): From 800df92382d0901fa139b06ae255b8085f32889c Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Tue, 10 Jan 2023 15:49:31 +0000 Subject: [PATCH 13/14] Add test_env_resolver_is_registered_after_loading Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 28 +++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 9520afb632..4f2cd7fbe1 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -452,10 +452,20 @@ def test_load_credentials_from_env_variables(self, tmp_path): assert conf["credentials"]["user"]["name"] == "test_user" assert conf["credentials"]["user"]["key"] == "test_key" + @use_config_dir + @use_catalog_env_variable_yml + def test_env_resolver_not_used_for_catalog(self, tmp_path): + """Check that the oc.env resolver is not used for catalog loading""" + conf = OmegaConfLoader(str(tmp_path)) + os.environ["TEST_DATASET"] = "test_dataset" + with pytest.raises(errors.UnsupportedInterpolationType): + conf["catalog"]["test"]["file_path"] + @use_config_dir @use_credentials_env_variable_yml - def test_env_resolver_is_cleared_after_credentials_loading(self, tmp_path): - """Check that the oc.env resolver is cleared after loading credentials""" + def test_env_resolver_is_cleared_after_loading(self, tmp_path): + """Check that the ``oc.env`` resolver is cleared after loading credentials + in the case that it was not registered beforehand.""" conf = OmegaConfLoader(str(tmp_path)) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" @@ -463,10 +473,12 @@ def test_env_resolver_is_cleared_after_credentials_loading(self, tmp_path): assert not OmegaConf.has_resolver("oc.env") @use_config_dir - @use_catalog_env_variable_yml - def test_env_resolver_not_used_for_catalog(self, tmp_path): - """Check that the oc.env resolver is not used for catalog loading""" + @use_credentials_env_variable_yml + def test_env_resolver_is_registered_after_loading(self, tmp_path): + """Check that the ``oc.env`` resolver is reigstered after loading credentials + in the case that it was registered beforehand""" conf = OmegaConfLoader(str(tmp_path)) - os.environ["TEST_DATASET"] = "test_dataset" - with pytest.raises(errors.UnsupportedInterpolationType): - conf["catalog"]["test"]["file_path"] + os.environ["TEST_USERNAME"] = "test_user" + os.environ["TEST_KEY"] = "test_key" + assert conf["credentials"]["user"]["name"] == "test_user" + assert not OmegaConf.has_resolver("oc.env") From 7e6727fd23566e8339413353a42f562c3652cc30 Mon Sep 17 00:00:00 2001 From: Jannic Holzer Date: Tue, 10 Jan 2023 17:26:12 +0000 Subject: [PATCH 14/14] Fix test_env_resolver_is_registered_after_loading Signed-off-by: Jannic Holzer --- tests/config/test_omegaconf_config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 4f2cd7fbe1..ffcd6e6e74 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -9,6 +9,7 @@ import pytest import yaml from omegaconf import OmegaConf, errors +from omegaconf.resolvers import oc from yaml.parser import ParserError from kedro.config import MissingConfigException, OmegaConfLoader @@ -475,10 +476,12 @@ def test_env_resolver_is_cleared_after_loading(self, tmp_path): @use_config_dir @use_credentials_env_variable_yml def test_env_resolver_is_registered_after_loading(self, tmp_path): - """Check that the ``oc.env`` resolver is reigstered after loading credentials + """Check that the ``oc.env`` resolver is registered after loading credentials in the case that it was registered beforehand""" conf = OmegaConfLoader(str(tmp_path)) + OmegaConf.register_new_resolver("oc.env", oc.env) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" - assert not OmegaConf.has_resolver("oc.env") + assert OmegaConf.has_resolver("oc.env") + OmegaConf.clear_resolver("oc.env")