From 5d96b710a2bf14bb3768438ee848997a7937caaa Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 22 Jun 2022 17:35:29 +0100 Subject: [PATCH 01/24] Put framework logging into LOGGING and guard configure_logging from automatic import Signed-off-by: Antony Milne --- kedro/config/config.py | 4 ++-- kedro/config/default_logger.py | 21 ++++++++++----------- kedro/framework/cli/cli.py | 7 ++----- kedro/framework/project/__init__.py | 10 ++++++---- kedro/framework/session/session.py | 12 ++++++------ kedro/runner/parallel_runner.py | 19 +++++++++---------- 6 files changed, 35 insertions(+), 38 deletions(-) diff --git a/kedro/config/config.py b/kedro/config/config.py index 2665a4b60c..9169fb5d8b 100644 --- a/kedro/config/config.py +++ b/kedro/config/config.py @@ -56,8 +56,8 @@ class ConfigLoader(AbstractConfigLoader): >>> conf_path = str(project_path / settings.CONF_SOURCE) >>> conf_loader = ConfigLoader(conf_source=conf_path, env="local") >>> - >>> conf_logging = conf_loader.get('logging*') - >>> logging.config.dictConfig(conf_logging) # set logging conf + >>> logging_config = conf_loader.get('logging*') + >>> logging.config.dictConfig(logging_config) # set logging conf >>> >>> conf_catalog = conf_loader.get('catalog*', 'catalog*/**') >>> conf_params = conf_loader.get('**/parameters.yml') diff --git a/kedro/config/default_logger.py b/kedro/config/default_logger.py index 11e692663e..c57301aada 100644 --- a/kedro/config/default_logger.py +++ b/kedro/config/default_logger.py @@ -1,23 +1,22 @@ """This module facilitates the loading of the default ``kedro.config`` for setting up the logging """ - -import logging.config import os import sys +from pathlib import Path import click import yaml from rich.traceback import install -CURRENT_DIR = os.path.dirname(__file__) +from kedro.framework.project import configure_logging + -with open(os.path.join(CURRENT_DIR, "logging.yml"), encoding="utf-8") as conf_file: - LOGGING_CONFIG = yaml.safe_load(conf_file.read()) - logging.config.dictConfig(LOGGING_CONFIG) - logging.captureWarnings(True) +def configure_default_logging(): + logging_config = yaml.safe_load((Path(__file__).parent / "logging.yml").read_text()) + configure_logging(logging_config) -# We suppress click here to hide tracebacks related to it conversely, -# kedro is not suppressed to show its tracebacks for easier debugging. -# sys.executable is used to get the kedro executable path to hide the top level traceback. -install(show_locals=True, suppress=[click, os.path.dirname(sys.executable)]) + # We suppress click here to hide tracebacks related to it conversely, + # kedro is not suppressed to show its tracebacks for easier debugging. + # sys.executable is used to get the kedro executable path to hide the top level traceback. + install(show_locals=True, suppress=[click, os.path.dirname(sys.executable)]) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 282ed814aa..9f1038bf7e 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -3,7 +3,6 @@ This module implements commands available from the kedro CLI. """ import importlib -import logging import sys import webbrowser from collections import defaultdict @@ -13,9 +12,8 @@ import click import importlib_metadata -# pylint: disable=unused-import -import kedro.config.default_logger # noqa from kedro import __version__ as version +from kedro.config.default_logger import configure_default_logging from kedro.framework.cli.catalog import catalog_cli from kedro.framework.cli.hooks import get_cli_hook_manager from kedro.framework.cli.jupyter import jupyter_cli @@ -42,8 +40,6 @@ v{version} """ -logger = logging.getLogger(__name__) - @click.group(context_settings=CONTEXT_SETTINGS, name="Kedro") @click.version_option(version, "--version", "-V", help="Show version and exit") @@ -210,6 +206,7 @@ def main(): # pragma: no cover """Main entry point. Look for a ``cli.py``, and, if found, add its commands to `kedro`'s before invoking the CLI. """ + configure_default_logging() _init_plugins() cli_collection = KedroCLI(project_path=Path.cwd()) cli_collection() diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 9bd369f8b9..369696ce92 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -180,7 +180,7 @@ def configure(self, pipelines_module: Optional[str] = None) -> None: PACKAGE_NAME = None -LOGGING = None +LOGGING_CONFIG = None settings = _ProjectSettings() @@ -206,10 +206,12 @@ def configure_project(package_name: str): def configure_logging(logging_config: Dict[str, Any]) -> None: - """Configure logging to make it available as a global variable.""" + """Configure logging and make it available as a global variable.""" logging.config.dictConfig(logging_config) - global LOGGING - LOGGING = logging_config + logging.captureWarnings(True) + + global LOGGING_CONFIG + LOGGING_CONFIG = logging_config def validate_settings(): diff --git a/kedro/framework/session/session.py b/kedro/framework/session/session.py index eca3c00fed..485c67d95a 100644 --- a/kedro/framework/session/session.py +++ b/kedro/framework/session/session.py @@ -172,27 +172,27 @@ def create( # pylint: disable=too-many-arguments return session def _get_logging_config(self) -> Dict[str, Any]: - conf_logging = self._get_config_loader().get( + logging_config = self._get_config_loader().get( "logging*", "logging*/**", "**/logging*" ) # turn relative paths in logging config into absolute path # before initialising loggers - conf_logging = _convert_paths_to_absolute_posix( - project_path=self._project_path, conf_dictionary=conf_logging + logging_config = _convert_paths_to_absolute_posix( + project_path=self._project_path, conf_dictionary=logging_config ) - return conf_logging + return logging_config def _setup_logging(self) -> None: """Register logging specified in logging directory.""" try: - conf_logging = self._get_logging_config() + logging_config = self._get_logging_config() except MissingConfigException: self._logger.debug( "No project logging configuration loaded; " "Kedro's default logging configuration will be used." ) else: - configure_logging(conf_logging) + configure_logging(logging_config) def _init_store(self) -> BaseSessionStore: store_class = settings.SESSION_STORE_CLASS diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index 208b9d974c..42cc47661e 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -1,7 +1,6 @@ """``ParallelRunner`` is an ``AbstractRunner`` implementation. It can be used to run the ``Pipeline`` in parallel groups formed by toposort. """ -import logging.config import multiprocessing import os import pickle @@ -81,14 +80,14 @@ class ParallelRunnerManager(SyncManager): def _bootstrap_subprocess( - package_name: str, conf_logging: Optional[Dict[str, Any]] = None + package_name: str, logging_config: Optional[Dict[str, Any]] = None ): # pylint: disable=import-outside-toplevel,cyclic-import - from kedro.framework.project import configure_project + from kedro.framework.project import configure_logging, configure_project configure_project(package_name) - if conf_logging is not None: - logging.config.dictConfig(conf_logging) + if logging_config is not None: + configure_logging(logging_config) def _run_node_synchronization( # pylint: disable=too-many-arguments @@ -97,7 +96,7 @@ def _run_node_synchronization( # pylint: disable=too-many-arguments is_async: bool = False, session_id: str = None, package_name: str = None, - conf_logging: Optional[Dict[str, Any]] = None, + logging_config: Optional[Dict[str, Any]] = None, ) -> Node: """Run a single `Node` with inputs from and outputs to the `catalog`. @@ -111,14 +110,14 @@ def _run_node_synchronization( # pylint: disable=too-many-arguments asynchronously with threads. Defaults to False. session_id: The session id of the pipeline run. package_name: The name of the project Python package. - conf_logging: A dictionary containing logging configuration. + logging_config: A dictionary containing logging configuration. Returns: The node argument. """ if multiprocessing.get_start_method() == "spawn" and package_name: # type: ignore - _bootstrap_subprocess(package_name, conf_logging) + _bootstrap_subprocess(package_name, logging_config) hook_manager = _create_hook_manager() _register_hooks(hook_manager, settings.HOOKS) @@ -300,7 +299,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression done = None max_workers = self._get_required_workers_count(pipeline) - from kedro.framework.project import LOGGING, PACKAGE_NAME + from kedro.framework.project import LOGGING_CONFIG, PACKAGE_NAME with ProcessPoolExecutor(max_workers=max_workers) as pool: while True: @@ -315,7 +314,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression self._is_async, session_id, package_name=PACKAGE_NAME, - conf_logging=LOGGING, + logging_config=LOGGING_CONFIG, ) ) if not futures: From a234f849a044c7dba471a49eb574a2e8b29406c8 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 22 Jun 2022 17:50:32 +0100 Subject: [PATCH 02/24] Fix tests Signed-off-by: Antony Milne --- tests/runner/test_parallel_runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/runner/test_parallel_runner.py b/tests/runner/test_parallel_runner.py index 9a3b86f04b..845eed8543 100644 --- a/tests/runner/test_parallel_runner.py +++ b/tests/runner/test_parallel_runner.py @@ -359,14 +359,14 @@ def mock_run_node(self, mocker): def mock_configure_project(self, mocker): return mocker.patch("kedro.framework.project.configure_project") - @pytest.mark.parametrize("conf_logging", [{"fake_logging_config": True}, {}]) + @pytest.mark.parametrize("logging_config", [{"fake_logging_config": True}, {}]) def test_package_name_and_logging_provided( self, mock_logging, mock_run_node, mock_configure_project, is_async, - conf_logging, + logging_config, mocker, ): mocker.patch("multiprocessing.get_start_method", return_value="spawn") @@ -381,10 +381,10 @@ def test_package_name_and_logging_provided( is_async, session_id, package_name=package_name, - conf_logging=conf_logging, + logging_config=logging_config, ) mock_run_node.assert_called_once() - mock_logging.assert_called_once_with(conf_logging) + mock_logging.assert_called_once_with(logging_config) mock_configure_project.assert_called_once_with(package_name) def test_package_name_provided( From 2b23ee1ec9bd807dde9fb7ccc45429570491e306 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Wed, 22 Jun 2022 17:51:49 +0100 Subject: [PATCH 03/24] Update kedro/config/config.py --- kedro/config/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro/config/config.py b/kedro/config/config.py index 9169fb5d8b..2665a4b60c 100644 --- a/kedro/config/config.py +++ b/kedro/config/config.py @@ -56,8 +56,8 @@ class ConfigLoader(AbstractConfigLoader): >>> conf_path = str(project_path / settings.CONF_SOURCE) >>> conf_loader = ConfigLoader(conf_source=conf_path, env="local") >>> - >>> logging_config = conf_loader.get('logging*') - >>> logging.config.dictConfig(logging_config) # set logging conf + >>> conf_logging = conf_loader.get('logging*') + >>> logging.config.dictConfig(conf_logging) # set logging conf >>> >>> conf_catalog = conf_loader.get('catalog*', 'catalog*/**') >>> conf_params = conf_loader.get('**/parameters.yml') From aa78c06d8ebf0377bd67da636c420cda2b18f3da Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 16:31:54 +0100 Subject: [PATCH 04/24] Take 2 with no default_logger file at all Signed-off-by: Antony Milne --- features/steps/test_plugin/plugin.py | 2 + kedro/config/default_logger.py | 22 --------- kedro/config/logging.yml | 14 ------ kedro/framework/cli/cli.py | 3 +- kedro/framework/project/__init__.py | 36 +++++++++++--- kedro/framework/project/default_logging.yml | 55 +++++++++++++++++++++ kedro/runner/parallel_runner.py | 13 ++--- 7 files changed, 92 insertions(+), 53 deletions(-) delete mode 100644 kedro/config/default_logger.py delete mode 100644 kedro/config/logging.yml create mode 100644 kedro/framework/project/default_logging.yml diff --git a/features/steps/test_plugin/plugin.py b/features/steps/test_plugin/plugin.py index 82b3e73832..0520d8da6b 100644 --- a/features/steps/test_plugin/plugin.py +++ b/features/steps/test_plugin/plugin.py @@ -3,6 +3,8 @@ from kedro.framework.hooks import hook_impl +from kedro.framework.cli.starters import _STARTERS_REPO + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/kedro/config/default_logger.py b/kedro/config/default_logger.py deleted file mode 100644 index c57301aada..0000000000 --- a/kedro/config/default_logger.py +++ /dev/null @@ -1,22 +0,0 @@ -"""This module facilitates the loading of the default ``kedro.config`` -for setting up the logging -""" -import os -import sys -from pathlib import Path - -import click -import yaml -from rich.traceback import install - -from kedro.framework.project import configure_logging - - -def configure_default_logging(): - logging_config = yaml.safe_load((Path(__file__).parent / "logging.yml").read_text()) - configure_logging(logging_config) - - # We suppress click here to hide tracebacks related to it conversely, - # kedro is not suppressed to show its tracebacks for easier debugging. - # sys.executable is used to get the kedro executable path to hide the top level traceback. - install(show_locals=True, suppress=[click, os.path.dirname(sys.executable)]) diff --git a/kedro/config/logging.yml b/kedro/config/logging.yml deleted file mode 100644 index 5c7b0fd798..0000000000 --- a/kedro/config/logging.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 1 - -disable_existing_loggers: False - -handlers: - rich: - class: rich.logging.RichHandler - -loggers: - kedro: - level: INFO - -root: - handlers: [rich] diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 9f1038bf7e..61ee779a5c 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -13,7 +13,7 @@ import importlib_metadata from kedro import __version__ as version -from kedro.config.default_logger import configure_default_logging +from kedro.framework.project import LOGGING_CONFIG from kedro.framework.cli.catalog import catalog_cli from kedro.framework.cli.hooks import get_cli_hook_manager from kedro.framework.cli.jupyter import jupyter_cli @@ -206,7 +206,6 @@ def main(): # pragma: no cover """Main entry point. Look for a ``cli.py``, and, if found, add its commands to `kedro`'s before invoking the CLI. """ - configure_default_logging() _init_plugins() cli_collection = KedroCLI(project_path=Path.cwd()) cli_collection() diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 369696ce92..f2c4c62e98 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -1,10 +1,18 @@ """``kedro.framework.project`` module provides utitlity to configure a Kedro project and access its settings.""" # pylint: disable=redefined-outer-name,unused-argument,global-statement +import sys +from collections import UserDict + +import click +from pathlib import Path + import importlib import logging.config import operator +import yaml from collections.abc import MutableMapping +from rich.traceback import install as rich_traceback_install from typing import Any, Dict, Optional from dynaconf import LazySettings @@ -179,8 +187,26 @@ def configure(self, pipelines_module: Optional[str] = None) -> None: __str__ = _load_data_wrapper(str) +class _ProjectLogging(UserDict): + def __init__(self): + default_logging = (Path(__file__).parent / "default_logging.yml").read_text() + self.configure(yaml.safe_load(default_logging)) + + # We suppress click here to hide tracebacks related to it conversely, + # kedro is not suppressed to show its tracebacks for easier debugging. + # sys.executable is used to get the kedro executable path to hide the top level traceback. + rich_traceback_install( + show_locals=True, suppress=[click, str(Path(sys.executable).parent)] + ) + logging.captureWarnings(True) + + def configure(self, logging_config: Dict[str, Any]) -> None: + logging.config.dictConfig(logging_config) + self.data = logging_config + + PACKAGE_NAME = None -LOGGING_CONFIG = None +LOGGING_CONFIG = _ProjectLogging() settings = _ProjectSettings() @@ -206,12 +232,8 @@ def configure_project(package_name: str): def configure_logging(logging_config: Dict[str, Any]) -> None: - """Configure logging and make it available as a global variable.""" - logging.config.dictConfig(logging_config) - logging.captureWarnings(True) - - global LOGGING_CONFIG - LOGGING_CONFIG = logging_config + """Configure logging according to `logging_config` dictionary.""" + LOGGING_CONFIG.configure(logging_config) def validate_settings(): diff --git a/kedro/framework/project/default_logging.yml b/kedro/framework/project/default_logging.yml new file mode 100644 index 0000000000..db4fc3f918 --- /dev/null +++ b/kedro/framework/project/default_logging.yml @@ -0,0 +1,55 @@ +version: 1 +disable_existing_loggers: False +formatters: + simple: + format: "FRAMEWORK %(asctime)s - %(name)s - %(levelname)s - %(message)s" + json_formatter: + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + class: pythonjsonlogger.jsonlogger.JsonFormatter + +handlers: + console: + class: logging.StreamHandler + level: INFO + formatter: simple + stream: ext://sys.stdout + + info_file_handler: + class: logging.handlers.RotatingFileHandler + level: INFO + formatter: simple + filename: logs/info.log + maxBytes: 10485760 # 10MB + backupCount: 20 + encoding: utf8 + delay: True + + error_file_handler: + class: logging.handlers.RotatingFileHandler + level: ERROR + formatter: simple + filename: logs/errors.log + maxBytes: 10485760 # 10MB + backupCount: 20 + encoding: utf8 + delay: True + +loggers: + anyconfig: + level: WARNING + handlers: [console, info_file_handler, error_file_handler] + propagate: no + + kedro.io: + level: INFO + handlers: [console, info_file_handler, error_file_handler] + propagate: no + + kedro.pipeline: + level: INFO + handlers: [console, info_file_handler, error_file_handler] + propagate: no + +root: + level: INFO + handlers: [console, info_file_handler, error_file_handler] diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index 42cc47661e..9cfe038cad 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -11,7 +11,7 @@ from multiprocessing.managers import BaseProxy, SyncManager # type: ignore from multiprocessing.reduction import ForkingPickler from pickle import PicklingError -from typing import Any, Dict, Iterable, Optional, Set +from typing import Any, Dict, Iterable, Set from pluggy import PluginManager @@ -79,15 +79,12 @@ class ParallelRunnerManager(SyncManager): ) -def _bootstrap_subprocess( - package_name: str, logging_config: Optional[Dict[str, Any]] = None -): +def _bootstrap_subprocess(package_name: str, logging_config: Dict[str, Any]): # pylint: disable=import-outside-toplevel,cyclic-import from kedro.framework.project import configure_logging, configure_project configure_project(package_name) - if logging_config is not None: - configure_logging(logging_config) + configure_logging(logging_config) def _run_node_synchronization( # pylint: disable=too-many-arguments @@ -96,7 +93,7 @@ def _run_node_synchronization( # pylint: disable=too-many-arguments is_async: bool = False, session_id: str = None, package_name: str = None, - logging_config: Optional[Dict[str, Any]] = None, + logging_config: Dict[str, Any] = None, ) -> Node: """Run a single `Node` with inputs from and outputs to the `catalog`. @@ -299,7 +296,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression done = None max_workers = self._get_required_workers_count(pipeline) - from kedro.framework.project import LOGGING_CONFIG, PACKAGE_NAME + from kedro.framework.project import PACKAGE_NAME, LOGGING_CONFIG with ProcessPoolExecutor(max_workers=max_workers) as pool: while True: From 1c5fa6cc22bbc0ff51ddaf056fb7a50d59b77bd8 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 16:34:47 +0100 Subject: [PATCH 05/24] Tidy Signed-off-by: Antony Milne --- features/steps/test_plugin/plugin.py | 2 - kedro/framework/project/__init__.py | 4 +- kedro/framework/project/default_logging.yml | 53 +++------------------ 3 files changed, 8 insertions(+), 51 deletions(-) diff --git a/features/steps/test_plugin/plugin.py b/features/steps/test_plugin/plugin.py index 0520d8da6b..82b3e73832 100644 --- a/features/steps/test_plugin/plugin.py +++ b/features/steps/test_plugin/plugin.py @@ -3,8 +3,6 @@ from kedro.framework.hooks import hook_impl -from kedro.framework.cli.starters import _STARTERS_REPO - logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index f2c4c62e98..d638e8a9f2 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -206,7 +206,7 @@ def configure(self, logging_config: Dict[str, Any]) -> None: PACKAGE_NAME = None -LOGGING_CONFIG = _ProjectLogging() +LOGGING = _ProjectLogging() settings = _ProjectSettings() @@ -233,7 +233,7 @@ def configure_project(package_name: str): def configure_logging(logging_config: Dict[str, Any]) -> None: """Configure logging according to `logging_config` dictionary.""" - LOGGING_CONFIG.configure(logging_config) + LOGGING.configure(logging_config) def validate_settings(): diff --git a/kedro/framework/project/default_logging.yml b/kedro/framework/project/default_logging.yml index db4fc3f918..5c7b0fd798 100644 --- a/kedro/framework/project/default_logging.yml +++ b/kedro/framework/project/default_logging.yml @@ -1,55 +1,14 @@ version: 1 + disable_existing_loggers: False -formatters: - simple: - format: "FRAMEWORK %(asctime)s - %(name)s - %(levelname)s - %(message)s" - json_formatter: - format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - class: pythonjsonlogger.jsonlogger.JsonFormatter handlers: - console: - class: logging.StreamHandler - level: INFO - formatter: simple - stream: ext://sys.stdout - - info_file_handler: - class: logging.handlers.RotatingFileHandler - level: INFO - formatter: simple - filename: logs/info.log - maxBytes: 10485760 # 10MB - backupCount: 20 - encoding: utf8 - delay: True - - error_file_handler: - class: logging.handlers.RotatingFileHandler - level: ERROR - formatter: simple - filename: logs/errors.log - maxBytes: 10485760 # 10MB - backupCount: 20 - encoding: utf8 - delay: True + rich: + class: rich.logging.RichHandler loggers: - anyconfig: - level: WARNING - handlers: [console, info_file_handler, error_file_handler] - propagate: no - - kedro.io: - level: INFO - handlers: [console, info_file_handler, error_file_handler] - propagate: no - - kedro.pipeline: - level: INFO - handlers: [console, info_file_handler, error_file_handler] - propagate: no + kedro: + level: INFO root: - level: INFO - handlers: [console, info_file_handler, error_file_handler] + handlers: [rich] From b72e916ea8754dd7798b3a191508c7aff0abbc92 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 16:40:56 +0100 Subject: [PATCH 06/24] More tidy Signed-off-by: Antony Milne --- kedro/runner/parallel_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index 9cfe038cad..f701ba9acb 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -296,7 +296,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression done = None max_workers = self._get_required_workers_count(pipeline) - from kedro.framework.project import PACKAGE_NAME, LOGGING_CONFIG + from kedro.framework.project import LOGGING, PACKAGE_NAME with ProcessPoolExecutor(max_workers=max_workers) as pool: while True: @@ -311,7 +311,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression self._is_async, session_id, package_name=PACKAGE_NAME, - logging_config=LOGGING_CONFIG, + logging_config=LOGGING, ) ) if not futures: From de4cb596300fa93c3908c207ee9731564bdeed60 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Fri, 24 Jun 2022 17:09:26 +0100 Subject: [PATCH 07/24] Apply suggestions from code review --- kedro/framework/cli/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 61ee779a5c..a903a498d6 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -13,7 +13,8 @@ import importlib_metadata from kedro import __version__ as version -from kedro.framework.project import LOGGING_CONFIG +# pylint: disable=unused-import +from kedro.framework.project import LOGGING_CONFIG # noqa from kedro.framework.cli.catalog import catalog_cli from kedro.framework.cli.hooks import get_cli_hook_manager from kedro.framework.cli.jupyter import jupyter_cli From 1f367799bb883e7f2e50faae7c05efdd97a71268 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 17:38:20 +0100 Subject: [PATCH 08/24] Fix up tests Signed-off-by: Antony Milne --- kedro/extras/extensions/ipython.py | 1 - kedro/framework/cli/cli.py | 3 +-- tests/__init__.py | 1 - tests/framework/cli/test_cli.py | 3 --- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/kedro/extras/extensions/ipython.py b/kedro/extras/extensions/ipython.py index 2706e3d12e..c47d62af17 100644 --- a/kedro/extras/extensions/ipython.py +++ b/kedro/extras/extensions/ipython.py @@ -41,7 +41,6 @@ def reload_kedro( Setting the path will also make it default for subsequent calls. """ - import kedro.config.default_logger # noqa: F401 # pylint: disable=unused-import from kedro.framework.cli import load_entry_points from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index a903a498d6..7fdc040d3a 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -13,8 +13,6 @@ import importlib_metadata from kedro import __version__ as version -# pylint: disable=unused-import -from kedro.framework.project import LOGGING_CONFIG # noqa from kedro.framework.cli.catalog import catalog_cli from kedro.framework.cli.hooks import get_cli_hook_manager from kedro.framework.cli.jupyter import jupyter_cli @@ -30,6 +28,7 @@ KedroCliError, load_entry_points, ) +from kedro.framework.project import LOGGING # noqa from kedro.framework.startup import _is_project, bootstrap_project LOGO = rf""" diff --git a/tests/__init__.py b/tests/__init__.py index 331f0e2357..e69de29bb2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -import kedro.config.default_logger # noqa diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index ecdc88139a..fff475b60f 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -8,9 +8,6 @@ import click from click.testing import CliRunner from pytest import fixture, mark, raises - -# pylint: disable=unused-import -import kedro.config.default_logger # noqa from kedro import __version__ as version from kedro.framework.cli import load_entry_points from kedro.framework.cli.catalog import catalog_cli From 880bc294b95e073d4480ec404775c403060a79aa Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 17:59:37 +0100 Subject: [PATCH 09/24] More fix tests Signed-off-by: Antony Milne --- MANIFEST.in | 2 +- docs/conf.py | 17 ++++++++--------- docs/source/logging/logging.md | 4 ++-- tests/runner/test_parallel_runner.py | 28 ++-------------------------- 4 files changed, 13 insertions(+), 38 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index afc3ea2a5b..2f24776c72 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,6 @@ include README.md include LICENSE.md include requirements.txt include test_requirements.txt -include kedro/config/logging.yml +include kedro/framework/project/logging.yml include kedro/extras/extensions/*.png recursive-include templates * diff --git a/docs/conf.py b/docs/conf.py index 9a9ffaaded..1d66e0925d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -206,6 +206,7 @@ # "anchor not found" but it's a valid selector for code examples "https://docs.delta.io/latest/delta-update.html#language-python", "https://github.com/kedro-org/kedro/blob/main/kedro_technical_charter.pdf", + "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/config/logging.yml", ] # retry before render a link broken (fix for "too many requests") @@ -259,9 +260,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, "Kedro.tex", "Kedro Documentation", "Kedro", "manual") -] +latex_documents = [(master_doc, "Kedro.tex", "Kedro Documentation", "Kedro", "manual")] # -- Options for manual page output ------------------------------------------ @@ -378,7 +377,7 @@ def autolink_replacements(what: str) -> List[Tuple[str, str, str]]: # first do plural only for classes replacements += [ ( - fr"``{obj}``s", + rf"``{obj}``s", f":{what}:`~{module}.{obj}`\\\\s", obj, ) @@ -387,7 +386,7 @@ def autolink_replacements(what: str) -> List[Tuple[str, str, str]]: # singular replacements += [ - (fr"``{obj}``", f":{what}:`~{module}.{obj}`", obj) for obj in objects + (rf"``{obj}``", f":{what}:`~{module}.{obj}`", obj) for obj in objects ] # Look for recognised class names/function names which are NOT @@ -396,13 +395,13 @@ def autolink_replacements(what: str) -> List[Tuple[str, str, str]]: if what == "class": # first do plural only for classes suggestions += [ - (fr"(? Date: Fri, 24 Jun 2022 20:46:42 +0100 Subject: [PATCH 10/24] Fix MANIFEST.in Signed-off-by: Antony Milne --- MANIFEST.in | 2 +- docs/source/logging/logging.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 2f24776c72..59a7c64ab2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,6 @@ include README.md include LICENSE.md include requirements.txt include test_requirements.txt -include kedro/framework/project/logging.yml +include kedro/framework/project/default_logging.yml include kedro/extras/extensions/*.png recursive-include templates * diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index f4cd0ed2e9..74097e4544 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -1,7 +1,7 @@ # Logging Kedro uses [Python's `logging` library](https://docs.python.org/3/library/logging.html). Configuration is provided as a dictionary according to the [Python logging configuration schema](https://docs.python.org/3/library/logging.config.html#logging-config-dictschema) in two places: -1. [Default configuration built into the Kedro framework](https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/logging.yml). This cannot be altered. +1. [Default configuration built into the Kedro framework](https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml). This cannot be altered. 2. Your project-side logging configuration. Every project generated using Kedro's CLI `kedro new` command includes a file `conf/base/logging.yml`. You can alter this configuration and provide different configurations for different run environment according to the [standard Kedro mechanism for handling configuration](../kedro_project_setup/configuration.md). ```{note} From f5ccde436f065cda56880d74b6a5cb1ab139e89e Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 22:30:56 +0100 Subject: [PATCH 11/24] Lint Signed-off-by: Antony Milne --- kedro/framework/project/__init__.py | 16 +++++++--------- kedro/runner/parallel_runner.py | 6 +++--- tests/framework/cli/test_cli.py | 1 + 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index d638e8a9f2..a3ce52a317 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -1,22 +1,20 @@ """``kedro.framework.project`` module provides utitlity to configure a Kedro project and access its settings.""" # pylint: disable=redefined-outer-name,unused-argument,global-statement -import sys -from collections import UserDict - -import click -from pathlib import Path - import importlib import logging.config import operator -import yaml +import sys +from collections import UserDict from collections.abc import MutableMapping -from rich.traceback import install as rich_traceback_install -from typing import Any, Dict, Optional +from pathlib import Path +from typing import Any, Dict, Optional, cast +import click +import yaml from dynaconf import LazySettings from dynaconf.validator import ValidationError, Validator +from rich.traceback import install as rich_traceback_install from kedro.pipeline import Pipeline diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index f701ba9acb..ee744d5871 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -113,8 +113,8 @@ def _run_node_synchronization( # pylint: disable=too-many-arguments The node argument. """ - if multiprocessing.get_start_method() == "spawn" and package_name: # type: ignore - _bootstrap_subprocess(package_name, logging_config) + if multiprocessing.get_start_method() == "spawn" and package_name: + _bootstrap_subprocess(package_name, logging_config) # type: ignore hook_manager = _create_hook_manager() _register_hooks(hook_manager, settings.HOOKS) @@ -311,7 +311,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression self._is_async, session_id, package_name=PACKAGE_NAME, - logging_config=LOGGING, + logging_config=LOGGING, # type: ignore ) ) if not futures: diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index fff475b60f..2f2805a1f7 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -8,6 +8,7 @@ import click from click.testing import CliRunner from pytest import fixture, mark, raises + from kedro import __version__ as version from kedro.framework.cli import load_entry_points from kedro.framework.cli.catalog import catalog_cli From 0abfd6a43702d4dcb1273f66ac083b29a7927f92 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 22:31:42 +0100 Subject: [PATCH 12/24] Fix linkcheck Signed-off-by: Antony Milne --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 1d66e0925d..b720a83d70 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -206,7 +206,7 @@ # "anchor not found" but it's a valid selector for code examples "https://docs.delta.io/latest/delta-update.html#language-python", "https://github.com/kedro-org/kedro/blob/main/kedro_technical_charter.pdf", - "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/config/logging.yml", + "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/config/default_logging.yml", ] # retry before render a link broken (fix for "too many requests") From e603ba2751ddf572096bbb8313bb360f7d913b0b Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Fri, 24 Jun 2022 22:38:52 +0100 Subject: [PATCH 13/24] Update RELEASE Signed-off-by: Antony Milne --- RELEASE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 511baae0ce..010cd2a818 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -31,6 +31,8 @@ * Required `cookiecutter>=2.1.1` to address a [known command injection vulnerability](https://security.snyk.io/vuln/SNYK-PYTHON-COOKIECUTTER-2414281). * The session store no longer fails if a username cannot be found with `getpass.getuser`. +## Minor breaking changes to the API +* The module `kedro.config.default_logger` no longer exists. ## Upcoming deprecations for Kedro 0.19.0 * `kedro.extras.ColorHandler` will be removed in 0.19.0. From f8708f2495508eaa17fe5918ca4bdc92855f817d Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 29 Jun 2022 14:47:28 +0100 Subject: [PATCH 14/24] Tidy Signed-off-by: Antony Milne --- kedro/framework/cli/cli.py | 2 +- kedro/framework/cli/micropkg.py | 4 ++-- kedro/framework/cli/starters.py | 4 +--- kedro/framework/project/__init__.py | 12 ++++++++++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 7fdc040d3a..d2e5dfa8eb 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -28,7 +28,7 @@ KedroCliError, load_entry_points, ) -from kedro.framework.project import LOGGING # noqa +from kedro.framework.project import LOGGING # noqa # pylint:disable=unused-import from kedro.framework.startup import _is_project, bootstrap_project LOGO = rf""" diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py index cd362fdbe3..ad0b72f206 100644 --- a/kedro/framework/cli/micropkg.py +++ b/kedro/framework/cli/micropkg.py @@ -62,7 +62,7 @@ def micropkg_cli(): # pragma: no cover @micropkg_cli.group() -def micropkg(): +def anmicropkg(): """Commands for working with micro-packages.""" @@ -255,7 +255,7 @@ def _package_micropkgs_from_manifest(metadata: ProjectMetadata) -> None: @click.pass_obj # this will pass the metadata as first argument def package_micropkg( metadata: ProjectMetadata, module_path, env, alias, destination, all_flag -): # pylint: disable=too-many-arguments +): """Package up a modular pipeline or micro-package as a Python source distribution.""" if not module_path and not all_flag: click.secho( diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 2e97bfcf5d..f82ba9c99a 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -79,9 +79,7 @@ def create_cli(): # pragma: no cover @click.option("--starter", "-s", "starter_name", help=STARTER_ARG_HELP) @click.option("--checkout", help=CHECKOUT_ARG_HELP) @click.option("--directory", help=DIRECTORY_ARG_HELP) -def new( - config_path, starter_name, checkout, directory, **kwargs -): # pylint: disable=unused-argument +def new(config_path, starter_name, checkout, directory, **kwargs): """Create a new kedro project.""" if checkout and not starter_name: raise KedroCliError("Cannot use the --checkout flag without a --starter value.") diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index a3ce52a317..bfc0a920ec 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -186,9 +186,15 @@ def configure(self, pipelines_module: Optional[str] = None) -> None: class _ProjectLogging(UserDict): + # pylint: disable=super-init-not-called def __init__(self): - default_logging = (Path(__file__).parent / "default_logging.yml").read_text() + """Initialise project logging with default configuration. Also enable + rich tracebacks.""" + default_logging = (Path(__file__).parent / "default_logging.yml").read_text( + encoding="utf-8" + ) self.configure(yaml.safe_load(default_logging)) + logging.captureWarnings(True) # We suppress click here to hide tracebacks related to it conversely, # kedro is not suppressed to show its tracebacks for easier debugging. @@ -196,9 +202,11 @@ def __init__(self): rich_traceback_install( show_locals=True, suppress=[click, str(Path(sys.executable).parent)] ) - logging.captureWarnings(True) def configure(self, logging_config: Dict[str, Any]) -> None: + """Configure project logging using `logging_config` (e.g. from project + logging.yml). We store this in the UserDict data so that it can be reconfigured + in _bootstrap_subprocess.""" logging.config.dictConfig(logging_config) self.data = logging_config From 35ca6f184ea830b23919a44dc3f160a5a168bbb2 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:51:04 +0100 Subject: [PATCH 15/24] Update RELEASE.md --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 010cd2a818..a28387e1a1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -32,7 +32,7 @@ * The session store no longer fails if a username cannot be found with `getpass.getuser`. ## Minor breaking changes to the API -* The module `kedro.config.default_logger` no longer exists. +* The module `kedro.config.default_logger` no longer exists; default logging configuration is now set automatically through `kedro.framework.project.LOGGING`. Unless you explicitly import `kedro.config.default_logger` you do not need to make any changes. ## Upcoming deprecations for Kedro 0.19.0 * `kedro.extras.ColorHandler` will be removed in 0.19.0. From 8a321c36a5de8a02a573079a274e7613168f512c Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 29 Jun 2022 14:52:33 +0100 Subject: [PATCH 16/24] More tidy Signed-off-by: Antony Milne --- kedro/extras/extensions/ipython.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro/extras/extensions/ipython.py b/kedro/extras/extensions/ipython.py index c47d62af17..2fa8ea5f25 100644 --- a/kedro/extras/extensions/ipython.py +++ b/kedro/extras/extensions/ipython.py @@ -40,8 +40,8 @@ def reload_kedro( """Line magic which reloads all Kedro default variables. Setting the path will also make it default for subsequent calls. """ - from kedro.framework.cli import load_entry_points + from kedro.framework.project import LOGGING # noqa # pylint:disable=unused-import from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession from kedro.framework.startup import bootstrap_project From 5c6312e84725f1cb80d86c79caacc01b79eaf023 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:53:22 +0100 Subject: [PATCH 17/24] Update kedro/framework/cli/micropkg.py --- kedro/framework/cli/micropkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py index ad0b72f206..a5a22da3cc 100644 --- a/kedro/framework/cli/micropkg.py +++ b/kedro/framework/cli/micropkg.py @@ -62,7 +62,7 @@ def micropkg_cli(): # pragma: no cover @micropkg_cli.group() -def anmicropkg(): +def micropkg(): """Commands for working with micro-packages.""" From 904b899f5dfc0ff960f2e2f528a4c2e04f919b1b Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 29 Jun 2022 14:54:13 +0100 Subject: [PATCH 18/24] Tidy Signed-off-by: Antony Milne --- kedro/framework/project/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index bfc0a920ec..22045743c8 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -8,7 +8,7 @@ from collections import UserDict from collections.abc import MutableMapping from pathlib import Path -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, Optional import click import yaml From 3ece394d9afb6190f74e53a755ef4a041b9fed6b Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 29 Jun 2022 15:29:44 +0100 Subject: [PATCH 19/24] Add tests Signed-off-by: Antony Milne --- tests/framework/cli/test_cli.py | 11 ----------- tests/framework/project/test_logging.py | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 tests/framework/project/test_logging.py diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index 2f2805a1f7..eb4a807e2f 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -1,4 +1,3 @@ -import logging from collections import namedtuple from itertools import cycle from pathlib import Path @@ -72,16 +71,6 @@ def fake_session(mocker): return mocked_session -class TestDefaultLogging: - def test_setup_root_logger(self): - root_logger = logging.getLogger() - assert "rich" in {handler.name for handler in root_logger.handlers} - - def test_setup_kedro_logger(self): - kedro_logger = logging.getLogger("kedro") - assert kedro_logger.level == logging.INFO - - class TestCliCommands: def test_cli(self): """Run `kedro` without arguments.""" diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py new file mode 100644 index 0000000000..2ddb21c3d5 --- /dev/null +++ b/tests/framework/project/test_logging.py @@ -0,0 +1,24 @@ +import logging +from kedro.framework.project import ( + LOGGING, + configure_logging, +) # noqa # pylint:disable=unused-import + + +def test_default_logging_config(): + assert LOGGING.data == { + "disable_existing_loggers": False, + "handlers": {"rich": {"class": "rich.logging.RichHandler"}}, + "loggers": {"kedro": {"level": "INFO"}}, + "root": {"handlers": ["rich"]}, + "version": 1, + } + assert "rich" in {handler.name for handler in logging.getLogger().handlers} + assert logging.getLogger("kedro").level == logging.INFO + + +def test_configure_logging(): + logging_config = {"version": 1, "loggers": {"kedro": {"level": "WARNING"}}} + configure_logging(logging_config) + assert LOGGING.data == logging_config + assert logging.getLogger("kedro").level == logging.WARNING From 70d9dd7dbd9198782f14ec5cdac3d27e48ca5d02 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 29 Jun 2022 15:41:17 +0100 Subject: [PATCH 20/24] Cleanup after tests Signed-off-by: Antony Milne --- tests/framework/project/test_logging.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 2ddb21c3d5..82100c7c2f 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -1,18 +1,29 @@ import logging +import pytest + from kedro.framework.project import ( LOGGING, configure_logging, ) # noqa # pylint:disable=unused-import +default_logging_config = { + "version": 1, + "disable_existing_loggers": False, + "handlers": {"rich": {"class": "rich.logging.RichHandler"}}, + "loggers": {"kedro": {"level": "INFO"}}, + "root": {"handlers": ["rich"]}, +} + + +@pytest.fixture(autouse=True) +def reset_logging(): + yield + configure_logging(default_logging_config) + + def test_default_logging_config(): - assert LOGGING.data == { - "disable_existing_loggers": False, - "handlers": {"rich": {"class": "rich.logging.RichHandler"}}, - "loggers": {"kedro": {"level": "INFO"}}, - "root": {"handlers": ["rich"]}, - "version": 1, - } + assert LOGGING.data == default_logging_config assert "rich" in {handler.name for handler in logging.getLogger().handlers} assert logging.getLogger("kedro").level == logging.INFO From 08cda7a23d3b2cc074835b5dd7da01daf1b657b7 Mon Sep 17 00:00:00 2001 From: Antony Milne Date: Wed, 29 Jun 2022 17:06:29 +0100 Subject: [PATCH 21/24] Lint Signed-off-by: Antony Milne --- tests/framework/project/test_logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 82100c7c2f..aaef52c5bd 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -1,11 +1,11 @@ import logging + import pytest -from kedro.framework.project import ( +from kedro.framework.project import ( # noqa # pylint:disable=unused-import LOGGING, configure_logging, -) # noqa # pylint:disable=unused-import - +) default_logging_config = { "version": 1, From 55110c00481124e63f5306902fa324614dc6dee8 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Wed, 29 Jun 2022 18:37:00 +0100 Subject: [PATCH 22/24] Update docs/conf.py --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b720a83d70..33cf21d739 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -206,7 +206,7 @@ # "anchor not found" but it's a valid selector for code examples "https://docs.delta.io/latest/delta-update.html#language-python", "https://github.com/kedro-org/kedro/blob/main/kedro_technical_charter.pdf", - "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/config/default_logging.yml", + "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", ] # retry before render a link broken (fix for "too many requests") From 46fa3ad0591c7970eba4aa91394cd202ea0cfd7e Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Wed, 29 Jun 2022 18:54:41 +0100 Subject: [PATCH 23/24] Update docs/source/logging/logging.md --- docs/source/logging/logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 74097e4544..5b4e7908a6 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -12,7 +12,7 @@ Framework-side and project-side logging configuration are loaded through subsequ ## Default framework-side logging configuration -Kedro's [default logging configuration](https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/config/default_logging.yml) defines a handler called `rich` that uses the [Rich logging handler](https://rich.readthedocs.io/en/stable/logging.html) to format messages. We also use the [Rich traceback handler](https://rich.readthedocs.io/en/stable/traceback.html) to render exceptions. +Kedro's [default logging configuration](https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml) defines a handler called `rich` that uses the [Rich logging handler](https://rich.readthedocs.io/en/stable/logging.html) to format messages. We also use the [Rich traceback handler](https://rich.readthedocs.io/en/stable/traceback.html) to render exceptions. By default, Python only shows logging messages at level `WARNING` and above. Kedro's logging configuration specifies that `INFO` level messages from Kedro should also be emitted. This makes it easier to track the progress of your pipeline when you perform a `kedro run`. From b269dcec7bfcea28e2c2af3bf8690e641513e0e6 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Wed, 29 Jun 2022 18:55:07 +0100 Subject: [PATCH 24/24] Update docs/conf.py --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 33cf21d739..60d9ae47e0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -205,7 +205,6 @@ "https://www.java.com/en/download/help/download_options.html", # "403 Client Error: Forbidden for url" # "anchor not found" but it's a valid selector for code examples "https://docs.delta.io/latest/delta-update.html#language-python", - "https://github.com/kedro-org/kedro/blob/main/kedro_technical_charter.pdf", "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", ]