diff --git a/RELEASE.md b/RELEASE.md index 205e18a750..ddc7e9ab57 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -64,15 +64,17 @@ * Removed the `--parallel` flag from `kedro run` in favour of `--runner=ParallelRunner`. The `-p` flag is now an alias for `--pipeline`. * Removed deprecated `CONF_SOURCE`, `package_name`, `pipeline`, `pipelines`, and `io` attributes from `KedroContext` as well as the deprecated `KedroContext.run` method. * Changed the behaviour of `kedro build-reqs` to compile requirements from `requirements.txt` instead of `requirements.in` and save them to `requirements.lock` instead of `requirements.txt`. +* Removed `ProjectHooks.register_catalog` `hook_spec` in favour of loading `DATA_CATALOG_CLASS` directly from `settings.py`. The default option for `DATA_CATALOG_CLASS` is now set to `kedro.io.DataCatalog`. ## Thanks for supporting contributions [Deepyaman Datta](https://github.com/deepyaman), [Lucas Jamar](https://github.com/lucasjamar), [Simon Brugman](https://github.com/sbrugman) ## Migration guide from Kedro 0.17.* to 0.18.* -* Please remove any existing `hook_impl` of the `register_config_loader` method from `ProjectHooks` (or custom alternatives). +* Please remove any existing `hook_impl` of the `register_config_loader` and `register_catalog` methods from `ProjectHooks` (or custom alternatives). * Populate `settings.py` with `CONFIG_LOADER_CLASS` set to your expected config loader class (for example `kedro.config.TemplatedConfigLoader` or custom implementation). If `CONFIG_LOADER_CLASS` value is not set, it will default to `kedro.config.ConfigLoader` at runtime. * Populate `settings.py` with `CONFIG_LOADER_ARGS` set to a dictionary with expected keyword arguments. If `CONFIG_LOADER_ARGS` is not set, it will default to an empty dictionary. +* Populate `settings.py` with `DATA_CATALOG_CLASS` set to your expected data catalog class. If `DATA_CATALOG_CLASS` is not set, it will default to `kedro.io.DataCatalog` at runtime. * Optional: You can now remove all `params:` prefix when supplying values to `parameters` argument in a `pipeline()` call. * If you're using `pandas.ExcelDataSet`, make sure you have `openpyxl` installed in your environment. Note that this is automatically pulled if you specify `kedro[pandas.ExcelDataSet]==0.18.0` in your `requirements.in`. You can uninstall `xlrd` if you were only using it for this dataset. * If you're using `pandas.ParquetDataSet`, please pass pandas saving arguments directly to `save_args` instead of nested in `from_pandas` (e.g. `save_args = {"preserve_index": False}` instead of `save_args = {"from_pandas": {"preserve_index": False}}`). diff --git a/docs/source/extend_kedro/hooks.md b/docs/source/extend_kedro/hooks.md index 08cfe1c3f6..10ca10cdf6 100644 --- a/docs/source/extend_kedro/hooks.md +++ b/docs/source/extend_kedro/hooks.md @@ -49,10 +49,9 @@ The naming convention for error hooks is `on__error`, in which: #### Registration Hooks -In addition, Kedro defines Hook specifications to register certain library components to be used with the project. This is where users can define their custom class implementations. Currently, the following Hook specifications are provided: +In addition, Kedro defines Hook specifications to register certain library components to be used with the project. This is where users can define their custom class implementations. Currently, the following Hook specification is provided: * `register_pipelines` -* `register_catalog` The naming convention for registration hooks is `register_`. diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py index 324ba3fcf8..27f83aea20 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py @@ -1,19 +1,5 @@ """Project hooks.""" -from typing import Any, Dict, Optional - -from kedro.framework.hooks import hook_impl -from kedro.io import DataCatalog class ProjectHooks: - @hook_impl - def register_catalog( - self, - catalog: Optional[Dict[str, Dict[str, Any]]], - credentials: Dict[str, Dict[str, Any]], - load_versions: Dict[str, str], - save_version: str, - ) -> DataCatalog: - return DataCatalog.from_config( - catalog, credentials, load_versions, save_version - ) + pass diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py index d69056d3fa..5ab5e1db0b 100644 --- a/kedro/framework/cli/project.py +++ b/kedro/framework/cli/project.py @@ -328,7 +328,7 @@ def activate_nbstripout( @click.option( "--params", type=str, default="", help=PARAMS_ARG_HELP, callback=_split_params ) -# pylint: disable=too-many-arguments,unused-argument,too-many-locals +# pylint: disable=too-many-arguments,unused-argument def run( tag, env, diff --git a/kedro/framework/context/context.py b/kedro/framework/context/context.py index b11a2fa8bc..5ee9f214d8 100644 --- a/kedro/framework/context/context.py +++ b/kedro/framework/context/context.py @@ -8,6 +8,7 @@ from kedro.config import ConfigLoader, MissingConfigException from kedro.framework.hooks import get_hook_manager +from kedro.framework.project import settings from kedro.io import DataCatalog from kedro.pipeline.pipeline import _transcode_split @@ -267,18 +268,12 @@ def _get_catalog( ) conf_creds = self._get_config_credentials() - hook_manager = get_hook_manager() - catalog = hook_manager.hook.register_catalog( # pylint: disable=no-member + catalog = settings.DATA_CATALOG_CLASS.from_config( catalog=conf_catalog, credentials=conf_creds, load_versions=load_versions, save_version=save_version, ) - if not isinstance(catalog, DataCatalog): - raise KedroContextError( - f"Expected an instance of `DataCatalog`, " - f"got `{type(catalog).__name__}` instead." - ) feed_dict = self._get_feed_dict() catalog.add_feed_dict(feed_dict) diff --git a/kedro/framework/hooks/specs.py b/kedro/framework/hooks/specs.py index 94cdbcfb58..32ac8a4e37 100644 --- a/kedro/framework/hooks/specs.py +++ b/kedro/framework/hooks/specs.py @@ -132,7 +132,6 @@ def on_node_error( # pylint: disable=too-many-arguments class PipelineSpecs: """Namespace that defines all specifications for a pipeline's lifecycle hooks.""" - # pylint: disable=missing-param-doc @hook_spec def before_pipeline_run( self, run_params: Dict[str, Any], pipeline: Pipeline, catalog: DataCatalog @@ -298,18 +297,3 @@ def register_pipelines(self) -> Dict[str, Pipeline]: """ pass - - @hook_spec(firstresult=True) - def register_catalog( # pylint: disable=too-many-arguments - self, - catalog: Optional[Dict[str, Dict[str, Any]]], - credentials: Dict[str, Dict[str, Any]], - load_versions: Dict[str, str], - save_version: str, - ) -> DataCatalog: - """Hook to be invoked to register a project's data catalog. - - Returns: - An instance of a ``DataCatalog``. - """ - pass diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 71b8843b0c..2342b022e4 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -49,7 +49,7 @@ class _ProjectSettings(LazySettings): _CONF_SOURCE = Validator("CONF_SOURCE", default="conf") _HOOKS = Validator("HOOKS", default=tuple()) - _CONTEXT_CLASS = Validator( + _CONTEXT_CLASS = _IsSubclassValidator( "CONTEXT_CLASS", default=_get_default_class("kedro.framework.context.KedroContext"), ) @@ -59,10 +59,13 @@ class _ProjectSettings(LazySettings): ) _SESSION_STORE_ARGS = Validator("SESSION_STORE_ARGS", default={}) _DISABLE_HOOKS_FOR_PLUGINS = Validator("DISABLE_HOOKS_FOR_PLUGINS", default=tuple()) - _CONFIG_LOADER_CLASS = Validator( + _CONFIG_LOADER_CLASS = _IsSubclassValidator( "CONFIG_LOADER_CLASS", default=_get_default_class("kedro.config.ConfigLoader") ) _CONFIG_LOADER_ARGS = Validator("CONFIG_LOADER_ARGS", default={}) + _DATA_CATALOG_CLASS = _IsSubclassValidator( + "DATA_CATALOG_CLASS", default=_get_default_class("kedro.io.DataCatalog") + ) def __init__(self, *args, **kwargs): @@ -76,6 +79,7 @@ def __init__(self, *args, **kwargs): self._DISABLE_HOOKS_FOR_PLUGINS, self._CONFIG_LOADER_CLASS, self._CONFIG_LOADER_ARGS, + self._DATA_CATALOG_CLASS, ] ) super().__init__(*args, **kwargs) diff --git a/kedro/framework/session/session.py b/kedro/framework/session/session.py index bd715accb7..dd9bfb97f1 100644 --- a/kedro/framework/session/session.py +++ b/kedro/framework/session/session.py @@ -271,20 +271,12 @@ def _get_config_loader(self) -> ConfigLoader: extra_params = self.store.get("extra_params") config_loader_class = settings.CONFIG_LOADER_CLASS - try: - return config_loader_class( - conf_source=str(self._project_path / settings.CONF_SOURCE), - env=env, - runtime_params=extra_params, - **settings.CONFIG_LOADER_ARGS, - ) - except TypeError as exc: - raise TypeError( - f"Expected an instance of `ConfigLoader`, " - f"got `{settings.CONFIG_LOADER_CLASS}` of class " - f"`{type(settings.CONFIG_LOADER_CLASS)}` instead.\n" - f"The provided `CONFIG_LOADER_ARGS were: {settings.CONFIG_LOADER_ARGS}" - ) from exc + return config_loader_class( + conf_source=str(self._project_path / settings.CONF_SOURCE), + env=env, + runtime_params=extra_params, + **settings.CONFIG_LOADER_ARGS, + ) def close(self): """Close the current session and save its store to disk diff --git a/kedro/io/data_catalog.py b/kedro/io/data_catalog.py index 709e191053..7e6945186e 100644 --- a/kedro/io/data_catalog.py +++ b/kedro/io/data_catalog.py @@ -177,7 +177,6 @@ def __init__( def _logger(self): return logging.getLogger(__name__) - # pylint: disable=too-many-arguments @classmethod def from_config( cls: Type, diff --git a/kedro/io/partitioned_dataset.py b/kedro/io/partitioned_dataset.py index e8e7df901d..2952dd633c 100644 --- a/kedro/io/partitioned_dataset.py +++ b/kedro/io/partitioned_dataset.py @@ -439,6 +439,7 @@ def _parse_checkpoint_config( if self._credentials: default_config[CREDENTIALS_KEY] = deepcopy(self._credentials) + # pylint: disable=consider-iterating-dictionary if CREDENTIALS_KEY in default_config.keys() & checkpoint_config.keys(): self._logger.warning( KEY_PROPAGATION_WARNING, diff --git a/kedro/pipeline/node.py b/kedro/pipeline/node.py index 46f35bfa43..4f97f92e6c 100644 --- a/kedro/pipeline/node.py +++ b/kedro/pipeline/node.py @@ -10,7 +10,7 @@ from warnings import warn -class Node: # pylint: disable=too-many-instance-attributes +class Node: """``Node`` is an auxiliary class facilitating the operations required to run user-provided functions as part of Kedro pipelines. """ diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py index 324ba3fcf8..0054110188 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/hooks.py @@ -6,14 +6,4 @@ class ProjectHooks: - @hook_impl - def register_catalog( - self, - catalog: Optional[Dict[str, Dict[str, Any]]], - credentials: Dict[str, Dict[str, Any]], - load_versions: Dict[str, str], - save_version: str, - ) -> DataCatalog: - return DataCatalog.from_config( - catalog, credentials, load_versions, save_version - ) + pass diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py index 5ebde0697c..24c9c77c24 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py @@ -22,9 +22,8 @@ # Define the configuration folder. Defaults to `conf` # CONF_SOURCE = "conf" -# Select the project ConfigLoader class here. +# Define the project ConfigLoader class here. # Defaults to kedro.config.ConfigLoader -# Define the config loader. Defaults to ConfigLoader. # from kedro.config import TemplatedConfigLoader # CONFIG_LOADER_CLASS = TemplatedConfigLoader @@ -35,3 +34,7 @@ # "base_env": "base", # "default_run_env": "local", # } + +# Define the project DataCatalog class here. +# Defaults to kedro.io.DataCatalog +# DATA_CATALOG_CLASS = DataCatalog diff --git a/tests/framework/context/test_context.py b/tests/framework/context/test_context.py index 65739f032b..8abd1002f0 100644 --- a/tests/framework/context/test_context.py +++ b/tests/framework/context/test_context.py @@ -1,6 +1,7 @@ import configparser import json import re +import textwrap from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath from typing import Any, Dict @@ -19,18 +20,23 @@ _update_nested_dict, _validate_layers_for_transcoding, ) -from kedro.framework.hooks import get_hook_manager, hook_impl +from kedro.framework.hooks import get_hook_manager from kedro.framework.project import ( - Validator, + ValidationError, _ProjectSettings, configure_project, pipelines, ) -from kedro.io import DataCatalog MOCK_PACKAGE_NAME = "mock_package_name" +class BadCatalog: # pylint: disable=too-few-public-methods + """ + Catalog class that doesn't subclass `DataCatalog`, for testing only. + """ + + def _write_yaml(filepath: Path, config: Dict): filepath.parent.mkdir(parents=True, exist_ok=True) yaml_str = yaml.dump(config) @@ -145,23 +151,23 @@ def prepare_project_dir(tmp_path, base_config, local_config, local_logging_confi _write_toml(tmp_path / "pyproject.toml", pyproject_toml_payload) -class RegistrationHooks: - @hook_impl - def register_catalog( - self, catalog, credentials, load_versions, save_version - ) -> DataCatalog: - return DataCatalog.from_config( - catalog, credentials, load_versions, save_version +@pytest.fixture +def mock_settings_file_bad_data_catalog_class(tmpdir): + mock_settings_file = tmpdir.join("mock_settings_file.py") + mock_settings_file.write( + textwrap.dedent( + f""" + from {__name__} import BadCatalog + DATA_CATALOG_CLASS = BadCatalog + """ ) - - -class MockSettings(_ProjectSettings): - _HOOKS = Validator("HOOKS", default=(RegistrationHooks(),)) + ) + return mock_settings_file @pytest.fixture(autouse=True) def mock_settings(mocker): - mocked_settings = MockSettings() + mocked_settings = _ProjectSettings() mocker.patch("kedro.framework.session.session.settings", mocked_settings) return mocker.patch("kedro.framework.project.settings", mocked_settings) @@ -271,6 +277,18 @@ def test_catalog(self, dummy_context, dummy_dataframe): reloaded_df = dummy_context.catalog.load("cars") assert_frame_equal(reloaded_df, dummy_dataframe) + def test_wrong_catalog_type(self, mock_settings_file_bad_data_catalog_class): + pattern = ( + "Invalid value `tests.framework.context.test_context.BadCatalog` received " + "for setting `DATA_CATALOG_CLASS`. " + "It must be a subclass of `kedro.io.data_catalog.DataCatalog`." + ) + mock_settings = _ProjectSettings( + settings_file=str(mock_settings_file_bad_data_catalog_class) + ) + with pytest.raises(ValidationError, match=re.escape(pattern)): + assert mock_settings.DATA_CATALOG_CLASS + @pytest.mark.parametrize( "extra_params", [None, {}, {"foo": "bar", "baz": [1, 2], "qux": None}], diff --git a/tests/framework/project/test_settings.py b/tests/framework/project/test_settings.py index b8cd016ef8..41748245ec 100644 --- a/tests/framework/project/test_settings.py +++ b/tests/framework/project/test_settings.py @@ -1,6 +1,5 @@ import sys import textwrap -from unittest import mock import pytest @@ -8,17 +7,9 @@ from kedro.framework.project import configure_project, settings from kedro.framework.session.store import BaseSessionStore -MOCK_CONTEXT_CLASS = mock.patch( - "kedro.framework.context.context.KedroContext", autospec=True -) - -def test_settings_without_configure_project_show_default_values(): - assert settings.CONF_SOURCE == "conf" - assert settings.CONTEXT_CLASS is KedroContext - assert settings.SESSION_STORE_CLASS is BaseSessionStore - assert settings.SESSION_STORE_ARGS == {} - assert len(settings.DISABLE_HOOKS_FOR_PLUGINS) == 0 +class MyContext(KedroContext): + pass @pytest.fixture @@ -28,9 +19,9 @@ def mock_package_name_with_settings_file(tmpdir): settings_file_path.write( textwrap.dedent( f""" - from {__name__} import MOCK_CONTEXT_CLASS + from {__name__} import MyContext CONF_SOURCE = "test_conf" - CONTEXT_CLASS = MOCK_CONTEXT_CLASS + CONTEXT_CLASS = MyContext """ ) ) @@ -43,9 +34,17 @@ def mock_package_name_with_settings_file(tmpdir): settings.set(key, value) +def test_settings_without_configure_project_show_default_values(): + assert settings.CONF_SOURCE == "conf" + assert settings.CONTEXT_CLASS is KedroContext + assert settings.SESSION_STORE_CLASS is BaseSessionStore + assert settings.SESSION_STORE_ARGS == {} + assert len(settings.DISABLE_HOOKS_FOR_PLUGINS) == 0 + + def test_settings_after_configuring_project_shows_updated_values( mock_package_name_with_settings_file, ): configure_project(mock_package_name_with_settings_file) assert settings.CONF_SOURCE == "test_conf" - assert settings.CONTEXT_CLASS is MOCK_CONTEXT_CLASS + assert settings.CONTEXT_CLASS is MyContext diff --git a/tests/framework/session/conftest.py b/tests/framework/session/conftest.py index 26747dddcf..7af110f7c1 100644 --- a/tests/framework/session/conftest.py +++ b/tests/framework/session/conftest.py @@ -2,7 +2,7 @@ from logging.handlers import QueueHandler, QueueListener from multiprocessing import Queue from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List import pandas as pd import pytest @@ -345,27 +345,6 @@ def after_dataset_saved(self, dataset_name: str, data: Any) -> None: "After dataset saved", extra={"dataset_name": dataset_name, "data": data} ) - @hook_impl - def register_catalog( - self, - catalog: Optional[Dict[str, Dict[str, Any]]], - credentials: Dict[str, Dict[str, Any]], - load_versions: Dict[str, str], - save_version: str, - ) -> DataCatalog: - logger.info( - "Registering catalog", - extra={ - "catalog": catalog, - "credentials": credentials, - "load_versions": load_versions, - "save_version": save_version, - }, - ) - return DataCatalog.from_config( - catalog, credentials, load_versions, save_version - ) - @pytest.fixture def project_hooks(): diff --git a/tests/framework/session/test_session.py b/tests/framework/session/test_session.py index ff5101ecb9..a7629103ee 100644 --- a/tests/framework/session/test_session.py +++ b/tests/framework/session/test_session.py @@ -14,6 +14,7 @@ from kedro.framework.project import ( ValidationError, Validator, + _IsSubclassValidator, _ProjectSettings, configure_project, ) @@ -30,6 +31,12 @@ class BadStore: # pylint: disable=too-few-public-methods """ +class BadConfigLoader: # pylint: disable=too-few-public-methods + """ + ConfigLoader class that doesn't subclass `ConfigLoader`, for testing only. + """ + + @pytest.fixture(autouse=True) def mocked_logging(mocker): # Disable logging.config.dictConfig in KedroSession._setup_logging as @@ -77,42 +84,33 @@ class MockSettings(_ProjectSettings): return _mock_imported_settings_paths(mocker, MockSettings()) -@pytest.fixture -def mock_config_loader_class(mocker): - return mocker.patch("kedro.config.ConfigLoader", autospec=True) - - -@pytest.fixture -def mock_settings_config_loader_class(mocker, mock_config_loader_class): - class MockSettings(_ProjectSettings): - _CONFIG_LOADER_CLASS = Validator( - "CONFIG_LOADER_CLASS", default=lambda *_: mock_config_loader_class - ) - - return _mock_imported_settings_paths(mocker, MockSettings()) - - -@pytest.fixture -def mock_settings_broken_config_loader_class(mocker): - class MockSettings(_ProjectSettings): - _CONFIG_LOADER_CLASS = Validator("CONFIG_LOADER_CLASS", default="it breaks") - - return _mock_imported_settings_paths(mocker, MockSettings()) - - @pytest.fixture def mock_settings_custom_config_loader_class(mocker): class MyConfigLoader(ConfigLoader): pass class MockSettings(_ProjectSettings): - _CONFIG_LOADER_CLASS = Validator( + _CONFIG_LOADER_CLASS = _IsSubclassValidator( "CONFIG_LOADER_CLASS", default=lambda *_: MyConfigLoader ) return _mock_imported_settings_paths(mocker, MockSettings()) +@pytest.fixture +def mock_settings_file_bad_config_loader_class(tmpdir): + mock_settings_file = tmpdir.join("mock_settings_file.py") + mock_settings_file.write( + textwrap.dedent( + f""" + from {__name__} import BadConfigLoader + CONFIG_LOADER_CLASS = BadConfigLoader + """ + ) + ) + return mock_settings_file + + @pytest.fixture def mock_settings_file_bad_session_store_class(tmpdir): mock_settings_file = tmpdir.join("mock_settings_file.py") @@ -153,7 +151,7 @@ def mock_settings_shelve_session_store(mocker, fake_project): shelve_location = fake_project / "nested" / "sessions" class MockSettings(_ProjectSettings): - _SESSION_STORE_CLASS = Validator( + _SESSION_STORE_CLASS = _IsSubclassValidator( "SESSION_STORE_CLASS", default=lambda *_: ShelveStore ) _SESSION_STORE_ARGS = Validator( @@ -235,14 +233,12 @@ class FakeException(Exception): class TestKedroSession: @pytest.mark.usefixtures("mock_settings_context_class") - @pytest.mark.usefixtures("mock_settings_config_loader_class") @pytest.mark.parametrize("env", [None, "env1"]) @pytest.mark.parametrize("extra_params", [None, {"key": "val"}]) def test_create( self, fake_project, mock_context_class, - mock_config_loader_class, fake_session_id, mock_package_name, mocker, @@ -250,6 +246,7 @@ def test_create( extra_params, ): mock_click_ctx = mocker.patch("click.get_current_context").return_value + mocker.patch("kedro.framework.session.KedroSession._get_logging_config") session = KedroSession.create( mock_package_name, fake_project, env=env, extra_params=extra_params ) @@ -273,15 +270,13 @@ def test_create( assert session.store == expected_store assert session.load_context() is mock_context_class.return_value - assert session._get_config_loader() is mock_config_loader_class.return_value + assert isinstance(session._get_config_loader(), ConfigLoader) @pytest.mark.usefixtures("mock_settings_context_class") - @pytest.mark.usefixtures("mock_settings_config_loader_class") def test_create_no_env_extra_params( self, fake_project, mock_context_class, - mock_config_loader_class, fake_session_id, mock_package_name, mocker, @@ -304,7 +299,7 @@ def test_create_no_env_extra_params( assert session.store == expected_store assert session.load_context() is mock_context_class.return_value - assert session._get_config_loader() is mock_config_loader_class.return_value + assert isinstance(session._get_config_loader(), ConfigLoader) @pytest.mark.usefixtures("mock_settings") def test_load_context_with_envvar( @@ -352,14 +347,17 @@ def test_load_config_loader_custom_config_loader_class( assert isinstance(result, ConfigLoader) assert result.__class__.__name__ == "MyConfigLoader" - @pytest.mark.usefixtures("mock_settings_broken_config_loader_class") - def test_broken_config_loader(self, fake_project, mock_package_name): + def test_broken_config_loader(self, mock_settings_file_bad_config_loader_class): pattern = ( - f"Expected an instance of `ConfigLoader`, " - f"got `it breaks` of class `{type('')}` instead." + "Invalid value `tests.framework.session.test_session.BadConfigLoader` received " + "for setting `CONFIG_LOADER_CLASS`. " + "It must be a subclass of `kedro.config.config.ConfigLoader`." ) - with pytest.raises(TypeError, match=re.escape(pattern)): - KedroSession.create(mock_package_name, fake_project) + mock_settings = _ProjectSettings( + settings_file=str(mock_settings_file_bad_config_loader_class) + ) + with pytest.raises(ValidationError, match=re.escape(pattern)): + assert mock_settings.CONFIG_LOADER_CLASS @pytest.mark.usefixtures("mock_settings_context_class") def test_default_store( diff --git a/tests/framework/session/test_session_registration_hooks.py b/tests/framework/session/test_session_registration_hooks.py index f541dda98b..d14d7a3f9f 100644 --- a/tests/framework/session/test_session_registration_hooks.py +++ b/tests/framework/session/test_session_registration_hooks.py @@ -1,15 +1,12 @@ import logging import re -from typing import Any, Dict, Optional import pytest from dynaconf.validator import Validator -from kedro.framework.context import KedroContextError from kedro.framework.hooks import hook_impl from kedro.framework.project import _ProjectSettings, pipelines from kedro.framework.session import KedroSession -from kedro.io import DataCatalog from tests.framework.session.conftest import ( _assert_hook_call_record_has_expected_parameters, _mock_imported_settings_paths, @@ -53,32 +50,6 @@ def mock_settings_duplicate_hooks(mocker, project_hooks, pipeline_registration_h ) -class RequiredRegistrationHooks: - """Mandatory registration hooks""" - - @hook_impl - def register_catalog( - self, - catalog: Optional[Dict[str, Dict[str, Any]]], - credentials: Dict[str, Dict[str, Any]], - load_versions: Dict[str, str], - save_version: str, - ) -> DataCatalog: - return DataCatalog.from_config( # pragma: no cover - catalog, credentials, load_versions, save_version - ) - - -@pytest.fixture -def mock_settings_broken_catalog_hooks(mocker): - class BrokenCatalogHooks(RequiredRegistrationHooks): - @hook_impl - def register_catalog(self): # pylint: disable=arguments-differ - return None - - return _mock_settings_with_hooks(mocker, hooks=(BrokenCatalogHooks(),)) - - @pytest.fixture def mock_session( mock_settings_with_pipeline_hooks, mock_package_name, tmp_path @@ -114,23 +85,6 @@ def test_register_pipelines_is_called( } assert pipelines == expected_pipelines - def test_register_catalog_is_called(self, mock_session, caplog): - context = mock_session.load_context() - catalog = context.catalog - assert isinstance(catalog, DataCatalog) - - relevant_records = [ - r for r in caplog.records if r.getMessage() == "Registering catalog" - ] - assert len(relevant_records) == 1 - - record = relevant_records[0] - assert record.catalog.keys() == {"cars", "boats"} - assert record.credentials == {"dev_s3": "foo"} - # save_version is only passed during a run, not on the property getter - assert record.save_version is None - assert record.load_versions is None - class TestDuplicatePipelineRegistration: """Test to make sure that if pipelines are defined in both registration hooks @@ -150,13 +104,3 @@ def test_register_pipelines_with_duplicate_entries( ) with pytest.warns(UserWarning, match=re.escape(pattern)): assert pipelines == expected_pipelines - - -class TestBrokenRegistrationHooks: - @pytest.mark.usefixtures("mock_settings_broken_catalog_hooks") - def test_broken_register_catalog_hook(self, tmp_path, mock_package_name): - pattern = "Expected an instance of `DataCatalog`, got `NoneType` instead." - with KedroSession.create(mock_package_name, tmp_path) as session: - context = session.load_context() - with pytest.raises(KedroContextError, match=re.escape(pattern)): - _ = context.catalog