From db214c24500ffc40a5b430ed17c8a48f9ebd7481 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 19 May 2023 17:39:51 +0100 Subject: [PATCH 01/41] Add the Kedro RichHandler Signed-off-by: Nok Chan --- kedro/extras/logging/__init__.py | 3 +- kedro/extras/logging/rich_logger.py | 48 +++++++++++++++++++++++++++++ kedro/framework/project/__init__.py | 12 +------- 3 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 kedro/extras/logging/rich_logger.py diff --git a/kedro/extras/logging/__init__.py b/kedro/extras/logging/__init__.py index 2892db7846..f33242dc43 100644 --- a/kedro/extras/logging/__init__.py +++ b/kedro/extras/logging/__init__.py @@ -4,8 +4,9 @@ import warnings from .color_logger import ColorHandler +from .rich_logger import RichHandler -__all__ = ["ColorHandler"] +__all__ = ["ColorHandler", "Richhandler"] warnings.simplefilter("default", DeprecationWarning) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py new file mode 100644 index 0000000000..f881713f3f --- /dev/null +++ b/kedro/extras/logging/rich_logger.py @@ -0,0 +1,48 @@ +import logging +from multiprocessing.sharedctypes import Value +import os +from pathlib import Path + +import click +import rich +import sys + +class RichHandler(rich.logging.RichHandler): + """Identical to rich's logging handler but with a few extra behaviours: + * warnings issued by the `warnings` module are redirected to logging + * pretty printing is enabled on the Python REPL (including IPython and Jupyter) + * all tracebacks are handled by rich when rich_tracebacks=True + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + logging.captureWarnings(True) + rich.pretty.install() + if self.rich_tracebacks: + # We suppress click here to hide tracebacks related to it conversely, + # kedro is not suppressed to show its tracebacks for easier debugging. + # sys.executable is used to get the kedro executable path to hide the + # top level traceback. + # Rich traceback handling does not work on databricks. Hopefully this will be + # fixed on their side at some point, but until then we disable it. + # See https://github.com/Textualize/rich/issues/2455 + + if "DATABRICKS_RUNTIME_VERSION" not in os.environ: + rich.traceback.install( + suppress=[ + click, + str(Path(sys.executable).parent) + ] + ) + ### ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ### NOTE YOU MAY WELL NEED TO CHANGE THIS PATH DEPENDING ON WHERE + ### THIS FILE ENDS UP SO THAT IT POINTS TO THE RIGHT PLACE + + +# class _ProjectLogging(Use Dict): +# # pylint: disable=super-init-not-called +# def __init__(self): +# """Initialise project logging. The path to logging configuration is given in +# environment variable KEDRO_LOGGING_CONFIG (defaults to default_logging.yml).""" +# path = os.environ.get("KEDRO_LOGGING_CONFIG", Path(__file__).parent / "default_logging.yml") +# logging_config = Path(path).read_text(encoding="utf-8") +# self.configure(yaml.safe_load(logging_config)) \ No newline at end of file diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 8a4922d0ba..2cd39849e7 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -112,7 +112,6 @@ class _ProjectSettings(LazySettings): ) def __init__(self, *args, **kwargs): - kwargs.update( validators=[ self._CONF_SOURCE, @@ -133,6 +132,7 @@ def _load_data_wrapper(func): """Wrap a method in _ProjectPipelines so that data is loaded on first access. Taking inspiration from dynaconf.utils.functional.new_method_proxy """ + # pylint: disable=protected-access def inner(self, *args, **kwargs): self._load_data() @@ -219,16 +219,6 @@ def __init__(self): ) self.configure(yaml.safe_load(default_logging)) logging.captureWarnings(True) - - # We suppress click here to hide tracebacks related to it conversely, - # kedro is not suppressed to show its tracebacks for easier debugging. - # sys.executable is used to get the kedro executable path to hide the - # top level traceback. - # Rich traceback handling does not work on databricks. Hopefully this will be - # fixed on their side at some point, but until then we disable it. - # See https://github.com/Textualize/rich/issues/2455 - if "DATABRICKS_RUNTIME_VERSION" not in os.environ: - rich.traceback.install(suppress=[click, str(Path(sys.executable).parent)]) rich.pretty.install() def configure(self, logging_config: Dict[str, Any]) -> None: From 6a31eff07d90131f856b6200f91ab894e90d3bec Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 13:58:32 +0100 Subject: [PATCH 02/41] Clean up imports and comments Signed-off-by: Nok Chan --- kedro/extras/logging/rich_logger.py | 37 ++++++++++++----------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index f881713f3f..aa55d18568 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -1,18 +1,22 @@ import logging -from multiprocessing.sharedctypes import Value import os +import sys +from multiprocessing.sharedctypes import Value from pathlib import Path import click -import rich -import sys +import rich.logging +import rich.pretty +import rich.traceback + class RichHandler(rich.logging.RichHandler): """Identical to rich's logging handler but with a few extra behaviours: - * warnings issued by the `warnings` module are redirected to logging - * pretty printing is enabled on the Python REPL (including IPython and Jupyter) - * all tracebacks are handled by rich when rich_tracebacks=True + * warnings issued by the `warnings` module are redirected to logging + * pretty printing is enabled on the Python REPL (including IPython and Jupyter) + * all tracebacks are handled by rich when rich_tracebacks=True """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logging.captureWarnings(True) @@ -28,21 +32,10 @@ def __init__(self, *args, **kwargs): if "DATABRICKS_RUNTIME_VERSION" not in os.environ: rich.traceback.install( - suppress=[ - click, - str(Path(sys.executable).parent) - ] - ) - ### ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ### NOTE YOU MAY WELL NEED TO CHANGE THIS PATH DEPENDING ON WHERE - ### THIS FILE ENDS UP SO THAT IT POINTS TO THE RIGHT PLACE + suppress=[click, str(Path(sys.executable).parent)] + ) -# class _ProjectLogging(Use Dict): -# # pylint: disable=super-init-not-called -# def __init__(self): -# """Initialise project logging. The path to logging configuration is given in -# environment variable KEDRO_LOGGING_CONFIG (defaults to default_logging.yml).""" -# path = os.environ.get("KEDRO_LOGGING_CONFIG", Path(__file__).parent / "default_logging.yml") -# logging_config = Path(path).read_text(encoding="utf-8") -# self.configure(yaml.safe_load(logging_config)) \ No newline at end of file +if __name__ == "__main__": + RichHandler() + print(1) From 2b02951f776b2dfe86f029e807638b327d833e5b Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 15:07:31 +0100 Subject: [PATCH 03/41] Map arguments to add flexibility to customize arguments available in RichHandler Signed-off-by: Nok Chan --- kedro/extras/logging/rich_logger.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index aa55d18568..874037c80f 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -21,6 +21,17 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logging.captureWarnings(True) rich.pretty.install() + # Map arguments from RichHandler's constructor to rich.traceback.install + # signatures + + mapped_kwargs = {} + for key, value in kwargs.items(): + prefix = "tracebacks_" + if key.startswith(prefix): + mapped_kwargs[key[len(prefix):]] = value + elif key in ("locals_max_length", "locals_max_string"): + mapped_kwargs[key] = value + if self.rich_tracebacks: # We suppress click here to hide tracebacks related to it conversely, # kedro is not suppressed to show its tracebacks for easier debugging. @@ -31,9 +42,12 @@ def __init__(self, *args, **kwargs): # See https://github.com/Textualize/rich/issues/2455 if "DATABRICKS_RUNTIME_VERSION" not in os.environ: - rich.traceback.install( - suppress=[click, str(Path(sys.executable).parent)] - ) + # rich.traceback.install( + # suppress=[click, str(Path(sys.executable).parent)] + # ) + # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler + # Support overlapping argument between RichHandler and rich.traceback.install + rich.traceback.install(**mapped_kwargs) if __name__ == "__main__": From df7308e0a6f3850a13cfee5aeb1b4670c3bc819c Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 16:01:52 +0100 Subject: [PATCH 04/41] Clean up rich_logger.py and unused imports Signed-off-by: Nok Chan --- kedro/extras/logging/rich_logger.py | 32 +++++++++++++---------------- kedro/framework/project/__init__.py | 6 +----- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index 874037c80f..de33d0cbe2 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -9,7 +9,7 @@ import rich.pretty import rich.traceback - +logger = logging.getLogger(__file__) class RichHandler(rich.logging.RichHandler): """Identical to rich's logging handler but with a few extra behaviours: * warnings issued by the `warnings` module are redirected to logging @@ -24,32 +24,28 @@ def __init__(self, *args, **kwargs): # Map arguments from RichHandler's constructor to rich.traceback.install # signatures - mapped_kwargs = {} + # We suppress click here to hide tracebacks related to it conversely, + # kedro is not suppressed to show its tracebacks for easier debugging. + # sys.executable is used to get the kedro executable path to hide the + # top level traceback. + # Rich traceback handling does not work on databricks. Hopefully this will be + # fixed on their side at some point, but until then we disable it. + # See https://github.com/Textualize/rich/issues/2455 + mapped_kwargs = {"suppress" : [click, str(Path(sys.executable).parent)]} for key, value in kwargs.items(): prefix = "tracebacks_" if key.startswith(prefix): - mapped_kwargs[key[len(prefix):]] = value + key_prefix_removed = key[len(prefix):] + if key_prefix_removed == "suppress": + mapped_kwargs[key_prefix_removed] += value + else: + mapped_kwargs[key_prefix_removed] = value elif key in ("locals_max_length", "locals_max_string"): mapped_kwargs[key] = value if self.rich_tracebacks: - # We suppress click here to hide tracebacks related to it conversely, - # kedro is not suppressed to show its tracebacks for easier debugging. - # sys.executable is used to get the kedro executable path to hide the - # top level traceback. - # Rich traceback handling does not work on databricks. Hopefully this will be - # fixed on their side at some point, but until then we disable it. - # See https://github.com/Textualize/rich/issues/2455 if "DATABRICKS_RUNTIME_VERSION" not in os.environ: - # rich.traceback.install( - # suppress=[click, str(Path(sys.executable).parent)] - # ) # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler # Support overlapping argument between RichHandler and rich.traceback.install rich.traceback.install(**mapped_kwargs) - - -if __name__ == "__main__": - RichHandler() - print(1) diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index dfdac1cfcf..81d10a487d 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -16,10 +16,7 @@ from pathlib import Path from typing import Any -import click import importlib_resources -import rich.pretty -import rich.traceback import yaml from dynaconf import LazySettings from dynaconf.validator import ValidationError, Validator @@ -221,8 +218,7 @@ def __init__(self): ) logging_config = Path(path).read_text(encoding="utf-8") self.configure(yaml.safe_load(logging_config)) - logging.captureWarnings(True) - rich.pretty.install() + def configure(self, logging_config: dict[str, Any]) -> None: """Configure project logging using ``logging_config`` (e.g. from project From c5079848aa5a04b878f91a9a554ca41076b7cf29 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 16:17:01 +0100 Subject: [PATCH 05/41] Add docs - first iteration Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 5260c1296e..af86de2bac 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -43,6 +43,22 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi + handlers: [console] ``` +### Customise `rich` handler +The `kedro.extras.logging.RichHandler` is subclass from [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler) and support the same set of arguments. By default we enable `rich_tracebacks: True` to use `rich` to render execeptions. You can turn it off with `rich_tracebacks: False`. + +When `rich_tracebacks` is set as `True`, the configuration will be propagate into +[`rich.pretty.install`](https://rich.readthedocs.io/en/stable/reference/pretty.html?highlight=rich.pretty.install#rich.pretty.install) +If the argument is supported in `rich.pretty.install`, it will also be propogated to the traceback's settings. + +For example, you can enable display of locals inside `logging.yml` to help debugging. +``` + rich: + class: kedro.extras.logging.RichHandler + rich_tracebacks: True + tracebacks_show_locals: True +``` + +The list of available options can be found in [Rich's documentation](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler) ### Use plain console logging To use plain rather than rich logging, swap the `rich` handler for the `console` one as follows: From 3ba89c9d199c70f5979f9e35aeaad9f881b2cfe5 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 16:17:25 +0100 Subject: [PATCH 06/41] Fix imports and small refactoring Signed-off-by: Nok Chan --- kedro/extras/logging/__init__.py | 2 +- kedro/extras/logging/rich_logger.py | 7 ++++--- kedro/framework/project/__init__.py | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kedro/extras/logging/__init__.py b/kedro/extras/logging/__init__.py index f33242dc43..1063002f5c 100644 --- a/kedro/extras/logging/__init__.py +++ b/kedro/extras/logging/__init__.py @@ -6,7 +6,7 @@ from .color_logger import ColorHandler from .rich_logger import RichHandler -__all__ = ["ColorHandler", "Richhandler"] +__all__ = ["ColorHandler", "RichHandler"] warnings.simplefilter("default", DeprecationWarning) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index de33d0cbe2..7dbbc7f6ab 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -10,6 +10,8 @@ import rich.traceback logger = logging.getLogger(__file__) + + class RichHandler(rich.logging.RichHandler): """Identical to rich's logging handler but with a few extra behaviours: * warnings issued by the `warnings` module are redirected to logging @@ -31,11 +33,11 @@ def __init__(self, *args, **kwargs): # Rich traceback handling does not work on databricks. Hopefully this will be # fixed on their side at some point, but until then we disable it. # See https://github.com/Textualize/rich/issues/2455 - mapped_kwargs = {"suppress" : [click, str(Path(sys.executable).parent)]} + mapped_kwargs = {"suppress": [click, str(Path(sys.executable).parent)]} for key, value in kwargs.items(): prefix = "tracebacks_" if key.startswith(prefix): - key_prefix_removed = key[len(prefix):] + key_prefix_removed = key[len(prefix) :] if key_prefix_removed == "suppress": mapped_kwargs[key_prefix_removed] += value else: @@ -44,7 +46,6 @@ def __init__(self, *args, **kwargs): mapped_kwargs[key] = value if self.rich_tracebacks: - if "DATABRICKS_RUNTIME_VERSION" not in os.environ: # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler # Support overlapping argument between RichHandler and rich.traceback.install diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 81d10a487d..7ffc1d882f 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -219,7 +219,6 @@ def __init__(self): logging_config = Path(path).read_text(encoding="utf-8") self.configure(yaml.safe_load(logging_config)) - def configure(self, logging_config: dict[str, Any]) -> None: """Configure project logging using ``logging_config`` (e.g. from project logging.yml). We store this in the UserDict data so that it can be reconfigured From 0fb70c3e726defdda9aad3cc98f6d9f4d89eefd0 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 16:20:40 +0100 Subject: [PATCH 07/41] Improve the doc Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index af86de2bac..7723bf3b24 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -43,22 +43,23 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi + handlers: [console] ``` -### Customise `rich` handler -The `kedro.extras.logging.RichHandler` is subclass from [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler) and support the same set of arguments. By default we enable `rich_tracebacks: True` to use `rich` to render execeptions. You can turn it off with `rich_tracebacks: False`. +### Customize the `rich` Handler -When `rich_tracebacks` is set as `True`, the configuration will be propagate into -[`rich.pretty.install`](https://rich.readthedocs.io/en/stable/reference/pretty.html?highlight=rich.pretty.install#rich.pretty.install) -If the argument is supported in `rich.pretty.install`, it will also be propogated to the traceback's settings. +Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. -For example, you can enable display of locals inside `logging.yml` to help debugging. -``` - rich: - class: kedro.extras.logging.RichHandler - rich_tracebacks: True - tracebacks_show_locals: True +When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.pretty.install`](https://rich.readthedocs.io/en/stable/reference/pretty.html?highlight=rich.pretty.install#rich.pretty.install). If an argument is compatible with `rich.pretty.install`, it will be passed to the traceback's settings. + +For instance, you can enable the display of local variables inside `logging.yml` to aid with debugging. + +```yaml +rich: + class: kedro.extras.logging.RichHandler + rich_tracebacks: True + tracebacks_show_locals: True ``` -The list of available options can be found in [Rich's documentation](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler) +A comprehensive list of available options can be found in the [Rich's documentation](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler). + ### Use plain console logging To use plain rather than rich logging, swap the `rich` handler for the `console` one as follows: From 2dbc0c0cd969a05f3777cd8aa73f05053b8792cb Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 16:22:44 +0100 Subject: [PATCH 08/41] Update the project's template Signed-off-by: Nok Chan --- .../{{ cookiecutter.repo_name }}/conf/base/logging.yml | 4 +++- kedro/framework/project/default_logging.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml index c8609b142b..1501f30063 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -34,7 +34,9 @@ handlers: delay: True rich: - class: rich.logging.RichHandler + class: kedro.extras.logging.RichHandler + rich_tracebacks: True + # tracebacks_show_locals: False loggers: kedro: diff --git a/kedro/framework/project/default_logging.yml b/kedro/framework/project/default_logging.yml index 5c7b0fd798..3328f74102 100644 --- a/kedro/framework/project/default_logging.yml +++ b/kedro/framework/project/default_logging.yml @@ -4,7 +4,9 @@ disable_existing_loggers: False handlers: rich: - class: rich.logging.RichHandler + class: kedro.extras.logging.RichHandler + rich_tracebacks: True + # tracebacks_show_locals: False loggers: kedro: From 05eaa910a475c5ec7f1215037247a448fd549079 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 16:32:03 +0100 Subject: [PATCH 09/41] format docstring Signed-off-by: Nok Chan --- kedro/extras/logging/rich_logger.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index 7dbbc7f6ab..7c6dc86727 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -23,8 +23,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logging.captureWarnings(True) rich.pretty.install() - # Map arguments from RichHandler's constructor to rich.traceback.install - # signatures # We suppress click here to hide tracebacks related to it conversely, # kedro is not suppressed to show its tracebacks for easier debugging. @@ -33,7 +31,10 @@ def __init__(self, *args, **kwargs): # Rich traceback handling does not work on databricks. Hopefully this will be # fixed on their side at some point, but until then we disable it. # See https://github.com/Textualize/rich/issues/2455 + mapped_kwargs = {"suppress": [click, str(Path(sys.executable).parent)]} + + # Mapping Arguments from RichHandler's Constructor to rich.traceback.install for key, value in kwargs.items(): prefix = "tracebacks_" if key.startswith(prefix): @@ -48,5 +49,5 @@ def __init__(self, *args, **kwargs): if self.rich_tracebacks: if "DATABRICKS_RUNTIME_VERSION" not in os.environ: # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler - # Support overlapping argument between RichHandler and rich.traceback.install + # Support compatible arguments between RichHandler and rich.traceback.install rich.traceback.install(**mapped_kwargs) From 7b9a989a5d83ff2f186476eb7432046a73eca4ee Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 17:02:57 +0100 Subject: [PATCH 10/41] Apply Jo's comment on the doc Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 7723bf3b24..b4584b1606 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -45,9 +45,9 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi ### Customize the `rich` Handler -Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. +Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. -When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.pretty.install`](https://rich.readthedocs.io/en/stable/reference/pretty.html?highlight=rich.pretty.install#rich.pretty.install). If an argument is compatible with `rich.pretty.install`, it will be passed to the traceback's settings. +When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.traceback.install`](https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install). If an argument is compatible with `rich.pretty.install`, it will be passed to the traceback's settings. For instance, you can enable the display of local variables inside `logging.yml` to aid with debugging. @@ -58,7 +58,7 @@ rich: tracebacks_show_locals: True ``` -A comprehensive list of available options can be found in the [Rich's documentation](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler). +A comprehensive list of available options can be found in the [RichHandler documentation](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler). ### Use plain console logging From 8e531f05e2aa4a30f6b5a5f269081aca6bd6e10d Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 23 May 2023 17:03:55 +0100 Subject: [PATCH 11/41] Fix the link with RichHandler Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index b4584b1606..ca0e55f63e 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -58,7 +58,7 @@ rich: tracebacks_show_locals: True ``` -A comprehensive list of available options can be found in the [RichHandler documentation](https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler). +A comprehensive list of available options can be found in the [RichHandler documentation](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler). ### Use plain console logging From 810d25be10e5678804b92570a245147babe5b01a Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Wed, 24 May 2023 11:43:22 +0100 Subject: [PATCH 12/41] Apply suggestions from code review Co-authored-by: Antony Milne <49395058+antonymilne@users.noreply.github.com> --- kedro/extras/logging/rich_logger.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index 7c6dc86727..d0a7b10f79 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -32,22 +32,21 @@ def __init__(self, *args, **kwargs): # fixed on their side at some point, but until then we disable it. # See https://github.com/Textualize/rich/issues/2455 - mapped_kwargs = {"suppress": [click, str(Path(sys.executable).parent)]} + traceback_install_kwargs = {"suppress": [click, str(Path(sys.executable).parent)]} # Mapping Arguments from RichHandler's Constructor to rich.traceback.install + prefix = "tracebacks_" for key, value in kwargs.items(): - prefix = "tracebacks_" if key.startswith(prefix): key_prefix_removed = key[len(prefix) :] if key_prefix_removed == "suppress": - mapped_kwargs[key_prefix_removed] += value + mapped_kwargs[key_prefix_removed].extend(value) else: mapped_kwargs[key_prefix_removed] = value elif key in ("locals_max_length", "locals_max_string"): mapped_kwargs[key] = value - if self.rich_tracebacks: - if "DATABRICKS_RUNTIME_VERSION" not in os.environ: + if self.rich_tracebacks and "DATABRICKS_RUNTIME_VERSION" not in os.environ: # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler # Support compatible arguments between RichHandler and rich.traceback.install rich.traceback.install(**mapped_kwargs) From 1e5d14a8233a9d75fe93eb84fecae15668a66c1c Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 11:58:06 +0100 Subject: [PATCH 13/41] Fix broken doc Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index ca0e55f63e..ff889b5454 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -45,9 +45,11 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi ### Customize the `rich` Handler -Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. +Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHand[label](vscode-file://vscode-a section of the class. -When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.traceback.install`](https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install). If an argument is compatible with `rich.pretty.install`, it will be passed to the traceback's settings. +pp/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html)ler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. + +When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.traceback.install`](https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install). If an argument is compatible with `rich.traceback.install`, it will be passed to the traceback's settings. For instance, you can enable the display of local variables inside `logging.yml` to aid with debugging. From 85ff28dea1e8ede72dc05a75ba7c7872aa5536c1 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 11:58:19 +0100 Subject: [PATCH 14/41] update template defaults Signed-off-by: Nok Chan --- .../{{ cookiecutter.repo_name }}/conf/base/logging.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml index 934e615283..1cfae089b7 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -24,7 +24,9 @@ handlers: delay: True rich: - class: rich.logging.RichHandler + class: kedro.extras.logging.RichHandler + rich_tracebacks: True + # tracebacks_show_locals: True loggers: kedro: From 1b0907882241fbde11616061bcc2ee2fa22cd369 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 11:58:38 +0100 Subject: [PATCH 15/41] refactoring RichHandler and improve doc Signed-off-by: Nok Chan --- kedro/extras/logging/rich_logger.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py index d0a7b10f79..f573ae1293 100644 --- a/kedro/extras/logging/rich_logger.py +++ b/kedro/extras/logging/rich_logger.py @@ -17,6 +17,13 @@ class RichHandler(rich.logging.RichHandler): * warnings issued by the `warnings` module are redirected to logging * pretty printing is enabled on the Python REPL (including IPython and Jupyter) * all tracebacks are handled by rich when rich_tracebacks=True + * constructor's arguments are mapped and pass to `rich.traceback.install` + + The list of available options of ``RichHandler`` can be found here: + https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler + + The list of available options of `rich.traceback.install` can be found here: + https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install """ def __init__(self, *args, **kwargs): @@ -32,7 +39,9 @@ def __init__(self, *args, **kwargs): # fixed on their side at some point, but until then we disable it. # See https://github.com/Textualize/rich/issues/2455 - traceback_install_kwargs = {"suppress": [click, str(Path(sys.executable).parent)]} + traceback_install_kwargs = { + "suppress": [click, str(Path(sys.executable).parent)] + } # Mapping Arguments from RichHandler's Constructor to rich.traceback.install prefix = "tracebacks_" @@ -40,13 +49,14 @@ def __init__(self, *args, **kwargs): if key.startswith(prefix): key_prefix_removed = key[len(prefix) :] if key_prefix_removed == "suppress": - mapped_kwargs[key_prefix_removed].extend(value) + traceback_install_kwargs[key_prefix_removed].extend(value) else: - mapped_kwargs[key_prefix_removed] = value + traceback_install_kwargs[key_prefix_removed] = value + # These two arguments are also compatible with RichHandler elif key in ("locals_max_length", "locals_max_string"): - mapped_kwargs[key] = value + traceback_install_kwargs[key] = value if self.rich_tracebacks and "DATABRICKS_RUNTIME_VERSION" not in os.environ: - # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler - # Support compatible arguments between RichHandler and rich.traceback.install - rich.traceback.install(**mapped_kwargs) + # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler + # Support compatible arguments between RichHandler and rich.traceback.install + rich.traceback.install(**traceback_install_kwargs) From f9dffc28f3d2eba06bfb3664a1cf40a69433cfa8 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 14:27:44 +0100 Subject: [PATCH 16/41] Fix broken test due to new default RichHandler Signed-off-by: Nok Chan --- tests/framework/project/test_logging.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 11cd7f08b3..75aa3ee872 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -11,7 +11,9 @@ default_logging_config = { "version": 1, "disable_existing_loggers": False, - "handlers": {"rich": {"class": "rich.logging.RichHandler"}}, + "handlers": { + "rich": {"class": "kedro.extras.logging.RichHandler", "rich_tracebacks": True} + }, "loggers": {"kedro": {"level": "INFO"}}, "root": {"handlers": ["rich"]}, } From f5be2d4a39b6732d0b4eec87e535305daa7bb5cd Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 16:08:42 +0100 Subject: [PATCH 17/41] Fix doc Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 2 +- kedro/extras/logging/rich_logger.py | 62 ----------------------------- 2 files changed, 1 insertion(+), 63 deletions(-) delete mode 100644 kedro/extras/logging/rich_logger.py diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index ff889b5454..efda45c899 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -19,7 +19,7 @@ We now give some common examples of how you might like to change your project's ### Using `KEDRO_LOGGING_CONFIG` environment variable -`KEDRO_LOGGING_CONFIG` is an optional environment variable that you can use to specify the path of your logging configuration file, overriding the default path of `conf/base/logging.yml`. +`KEDRO_LOGGING_CONFIG` is an optional environment variable that you can use to specify the path of your logging configuration file, overriding the default Kedro's framework `default_logging.yml`. To use this environment variable, set it to the path of your desired logging configuration file before running any Kedro commands. For example, if you have a logging configuration file located at `/path/to/logging.yml`, you can set `KEDRO_LOGGING_CONFIG` as follows: diff --git a/kedro/extras/logging/rich_logger.py b/kedro/extras/logging/rich_logger.py deleted file mode 100644 index f573ae1293..0000000000 --- a/kedro/extras/logging/rich_logger.py +++ /dev/null @@ -1,62 +0,0 @@ -import logging -import os -import sys -from multiprocessing.sharedctypes import Value -from pathlib import Path - -import click -import rich.logging -import rich.pretty -import rich.traceback - -logger = logging.getLogger(__file__) - - -class RichHandler(rich.logging.RichHandler): - """Identical to rich's logging handler but with a few extra behaviours: - * warnings issued by the `warnings` module are redirected to logging - * pretty printing is enabled on the Python REPL (including IPython and Jupyter) - * all tracebacks are handled by rich when rich_tracebacks=True - * constructor's arguments are mapped and pass to `rich.traceback.install` - - The list of available options of ``RichHandler`` can be found here: - https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler - - The list of available options of `rich.traceback.install` can be found here: - https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - logging.captureWarnings(True) - rich.pretty.install() - - # We suppress click here to hide tracebacks related to it conversely, - # kedro is not suppressed to show its tracebacks for easier debugging. - # sys.executable is used to get the kedro executable path to hide the - # top level traceback. - # Rich traceback handling does not work on databricks. Hopefully this will be - # fixed on their side at some point, but until then we disable it. - # See https://github.com/Textualize/rich/issues/2455 - - traceback_install_kwargs = { - "suppress": [click, str(Path(sys.executable).parent)] - } - - # Mapping Arguments from RichHandler's Constructor to rich.traceback.install - prefix = "tracebacks_" - for key, value in kwargs.items(): - if key.startswith(prefix): - key_prefix_removed = key[len(prefix) :] - if key_prefix_removed == "suppress": - traceback_install_kwargs[key_prefix_removed].extend(value) - else: - traceback_install_kwargs[key_prefix_removed] = value - # These two arguments are also compatible with RichHandler - elif key in ("locals_max_length", "locals_max_string"): - traceback_install_kwargs[key] = value - - if self.rich_tracebacks and "DATABRICKS_RUNTIME_VERSION" not in os.environ: - # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler - # Support compatible arguments between RichHandler and rich.traceback.install - rich.traceback.install(**traceback_install_kwargs) From 044ccdaf1dac68c5405261ae415165442d371eb2 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 16:11:38 +0100 Subject: [PATCH 18/41] Move kedro.extras.logging.RichHandler -> kedro.logging.RichHandler Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 2 +- .../conf/base/logging.yml | 2 +- kedro/extras/logging/__init__.py | 4 +- kedro/framework/project/default_logging.yml | 2 +- kedro/logging.py | 62 +++++++++++++++++++ .../conf/base/logging.yml | 2 +- tests/framework/project/test_logging.py | 11 ++-- 7 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 kedro/logging.py diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index efda45c899..7edf35741d 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -45,7 +45,7 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi ### Customize the `rich` Handler -Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHand[label](vscode-file://vscode-a section of the class. +Kedro's `kedro.logging.RichHandler` is a subclass of [`rich.logging.RichHand[label](vscode-file://vscode-a section of the class. pp/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html)ler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml index 1501f30063..68e9762299 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -34,7 +34,7 @@ handlers: delay: True rich: - class: kedro.extras.logging.RichHandler + class: kedro.logging.RichHandler rich_tracebacks: True # tracebacks_show_locals: False diff --git a/kedro/extras/logging/__init__.py b/kedro/extras/logging/__init__.py index 1063002f5c..856077b435 100644 --- a/kedro/extras/logging/__init__.py +++ b/kedro/extras/logging/__init__.py @@ -4,9 +4,9 @@ import warnings from .color_logger import ColorHandler -from .rich_logger import RichHandler +from logging import RichHandler -__all__ = ["ColorHandler", "RichHandler"] +__all__ = ["ColorHandler"] warnings.simplefilter("default", DeprecationWarning) diff --git a/kedro/framework/project/default_logging.yml b/kedro/framework/project/default_logging.yml index 3328f74102..a89e348087 100644 --- a/kedro/framework/project/default_logging.yml +++ b/kedro/framework/project/default_logging.yml @@ -4,7 +4,7 @@ disable_existing_loggers: False handlers: rich: - class: kedro.extras.logging.RichHandler + class: kedro.logging.RichHandler rich_tracebacks: True # tracebacks_show_locals: False diff --git a/kedro/logging.py b/kedro/logging.py new file mode 100644 index 0000000000..f573ae1293 --- /dev/null +++ b/kedro/logging.py @@ -0,0 +1,62 @@ +import logging +import os +import sys +from multiprocessing.sharedctypes import Value +from pathlib import Path + +import click +import rich.logging +import rich.pretty +import rich.traceback + +logger = logging.getLogger(__file__) + + +class RichHandler(rich.logging.RichHandler): + """Identical to rich's logging handler but with a few extra behaviours: + * warnings issued by the `warnings` module are redirected to logging + * pretty printing is enabled on the Python REPL (including IPython and Jupyter) + * all tracebacks are handled by rich when rich_tracebacks=True + * constructor's arguments are mapped and pass to `rich.traceback.install` + + The list of available options of ``RichHandler`` can be found here: + https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler + + The list of available options of `rich.traceback.install` can be found here: + https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + logging.captureWarnings(True) + rich.pretty.install() + + # We suppress click here to hide tracebacks related to it conversely, + # kedro is not suppressed to show its tracebacks for easier debugging. + # sys.executable is used to get the kedro executable path to hide the + # top level traceback. + # Rich traceback handling does not work on databricks. Hopefully this will be + # fixed on their side at some point, but until then we disable it. + # See https://github.com/Textualize/rich/issues/2455 + + traceback_install_kwargs = { + "suppress": [click, str(Path(sys.executable).parent)] + } + + # Mapping Arguments from RichHandler's Constructor to rich.traceback.install + prefix = "tracebacks_" + for key, value in kwargs.items(): + if key.startswith(prefix): + key_prefix_removed = key[len(prefix) :] + if key_prefix_removed == "suppress": + traceback_install_kwargs[key_prefix_removed].extend(value) + else: + traceback_install_kwargs[key_prefix_removed] = value + # These two arguments are also compatible with RichHandler + elif key in ("locals_max_length", "locals_max_string"): + traceback_install_kwargs[key] = value + + if self.rich_tracebacks and "DATABRICKS_RUNTIME_VERSION" not in os.environ: + # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler + # Support compatible arguments between RichHandler and rich.traceback.install + rich.traceback.install(**traceback_install_kwargs) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml index 1cfae089b7..137f193c18 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -24,7 +24,7 @@ handlers: delay: True rich: - class: kedro.extras.logging.RichHandler + class: kedro.logging.RichHandler rich_tracebacks: True # tracebacks_show_locals: True diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 75aa3ee872..9829173306 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -52,13 +52,10 @@ def test_configure_logging(): def test_rich_traceback_enabled(mocker): - """Note we need to force reload; just doing from kedro.framework.project import ... - will not call rich.traceback.install again. Using importlib.reload does not work - well with other tests here, hence the manual del sys.modules.""" rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") - del sys.modules["kedro.framework.project"] - from kedro.framework.project import LOGGING # noqa + + LOGGING.configure(default_logging_config) rich_traceback_install.assert_called() rich_pretty_install.assert_called() @@ -68,8 +65,8 @@ def test_rich_traceback_disabled_on_databricks(mocker, monkeypatch): monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "1") rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") - del sys.modules["kedro.framework.project"] - from kedro.framework.project import LOGGING # noqa + + LOGGING.configure(default_logging_config) rich_traceback_install.assert_not_called() rich_pretty_install.assert_called() From 23f64277c729e3949c2461862497033ac48df4dc Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 16:48:22 +0100 Subject: [PATCH 19/41] add kedro/logging.py and remove the unnecessary import that cause namespace collision Signed-off-by: Nok Chan --- kedro/__init__.py | 5 +---- kedro/logging.py | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/kedro/__init__.py b/kedro/__init__.py index 2de9928d4b..6aa7df727b 100644 --- a/kedro/__init__.py +++ b/kedro/__init__.py @@ -5,7 +5,4 @@ __version__ = "0.18.8" - -import logging - -logging.getLogger(__name__).addHandler(logging.NullHandler()) +import kedro.logging \ No newline at end of file diff --git a/kedro/logging.py b/kedro/logging.py index f573ae1293..b32232a310 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -1,7 +1,6 @@ import logging import os import sys -from multiprocessing.sharedctypes import Value from pathlib import Path import click From 04571eeee0ff1e188b940862323a361dd8fa4dfb Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 16:51:52 +0100 Subject: [PATCH 20/41] Refactor test Signed-off-by: Nok Chan --- tests/framework/project/test_logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 9829173306..a843e0a924 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -12,7 +12,7 @@ "version": 1, "disable_existing_loggers": False, "handlers": { - "rich": {"class": "kedro.extras.logging.RichHandler", "rich_tracebacks": True} + "rich": {"class": "kedro.logging.RichHandler", "rich_tracebacks": True} }, "loggers": {"kedro": {"level": "INFO"}}, "root": {"handlers": ["rich"]}, @@ -37,8 +37,8 @@ def test_environment_variable_logging_config(monkeypatch, tmp_path): logging_config = {"version": 1, "loggers": {"kedro": {"level": "WARNING"}}} with config_path.open("w", encoding="utf-8") as f: yaml.dump(logging_config, f) - del sys.modules["kedro.framework.project"] - from kedro.framework.project import LOGGING # noqa + from kedro.framework.project import _ProjectLogging + LOGGING = _ProjectLogging() assert LOGGING.data == logging_config assert logging.getLogger("kedro").level == logging.WARNING From ea23b38791ae54a442ad7274ab722383f9f7ffde Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 17:31:47 +0100 Subject: [PATCH 21/41] Add more tests Signed-off-by: Nok Chan --- tests/framework/project/test_logging.py | 44 ++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index a843e0a924..4e426aa917 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -1,4 +1,4 @@ -# pylint: disable=import-outside-toplevel,unused-import,reimported +# pylint: disable=import-outside-toplevel import logging import sys from pathlib import Path @@ -38,6 +38,7 @@ def test_environment_variable_logging_config(monkeypatch, tmp_path): with config_path.open("w", encoding="utf-8") as f: yaml.dump(logging_config, f) from kedro.framework.project import _ProjectLogging + LOGGING = _ProjectLogging() assert LOGGING.data == logging_config @@ -61,6 +62,47 @@ def test_rich_traceback_enabled(mocker): rich_pretty_install.assert_called() +def test_rich_traceback_not_installed(mocker): + rich_traceback_install = mocker.patch("rich.traceback.install") + rich_pretty_install = mocker.patch("rich.pretty.install") + rich_handler = { + "class": "kedro.logging.RichHandler", + "rich_tracebacks": False, + "tracebacks_show_locals": True, + } + test_logging_config = default_logging_config.copy() + test_logging_config["handlers"]["rich"] = rich_handler + + LOGGING.configure(test_logging_config) + + rich_pretty_install.assert_called_once() + rich_traceback_install.assert_not_called() + + +def test_rich_traceback_configuration(mocker): + rich_traceback_install = mocker.patch("rich.traceback.install") + rich_pretty_install = mocker.patch("rich.pretty.install") + import click + + sys_executable_path = str(Path(sys.executable).parent) + traceback_install_defaults = {"suppress": [click, sys_executable_path]} + + rich_handler = { + "class": "kedro.logging.RichHandler", + "rich_tracebacks": True, + "tracebacks_show_locals": True, + } + + test_logging_config = default_logging_config.copy() + test_logging_config["handlers"]["rich"] = rich_handler + LOGGING.configure(test_logging_config) + + rich_traceback_install.assert_called_with( + **{"show_locals": True, **traceback_install_defaults} + ) + rich_pretty_install.assert_called_once() + + def test_rich_traceback_disabled_on_databricks(mocker, monkeypatch): monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "1") rich_traceback_install = mocker.patch("rich.traceback.install") From 8894aa549759da1659f77d808f58dae9f34f678b Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 17:35:29 +0100 Subject: [PATCH 22/41] Fix lots of import, docstring Signed-off-by: Nok Chan --- kedro/__init__.py | 2 -- kedro/extras/logging/__init__.py | 1 - kedro/framework/project/__init__.py | 1 - kedro/logging.py | 4 ++++ 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kedro/__init__.py b/kedro/__init__.py index 6aa7df727b..0c11f5793f 100644 --- a/kedro/__init__.py +++ b/kedro/__init__.py @@ -4,5 +4,3 @@ """ __version__ = "0.18.8" - -import kedro.logging \ No newline at end of file diff --git a/kedro/extras/logging/__init__.py b/kedro/extras/logging/__init__.py index 856077b435..2892db7846 100644 --- a/kedro/extras/logging/__init__.py +++ b/kedro/extras/logging/__init__.py @@ -4,7 +4,6 @@ import warnings from .color_logger import ColorHandler -from logging import RichHandler __all__ = ["ColorHandler"] diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 7ffc1d882f..0d6946555e 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -7,7 +7,6 @@ import logging.config import operator import os -import sys import traceback import types import warnings diff --git a/kedro/logging.py b/kedro/logging.py index b32232a310..ae68aa9418 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -1,3 +1,7 @@ +""" +This module contains a logging handler class which produces coloured logs and tracebacks. +""" + import logging import os import sys From 5efefd80cc5ac10ca8d9443fa8ff0ee721b2d1e7 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 17:41:04 +0100 Subject: [PATCH 23/41] Reorder docstring Signed-off-by: Nok Chan --- kedro/logging.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kedro/logging.py b/kedro/logging.py index ae68aa9418..b8d665fc3c 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -38,9 +38,6 @@ def __init__(self, *args, **kwargs): # kedro is not suppressed to show its tracebacks for easier debugging. # sys.executable is used to get the kedro executable path to hide the # top level traceback. - # Rich traceback handling does not work on databricks. Hopefully this will be - # fixed on their side at some point, but until then we disable it. - # See https://github.com/Textualize/rich/issues/2455 traceback_install_kwargs = { "suppress": [click, str(Path(sys.executable).parent)] @@ -60,6 +57,7 @@ def __init__(self, *args, **kwargs): traceback_install_kwargs[key] = value if self.rich_tracebacks and "DATABRICKS_RUNTIME_VERSION" not in os.environ: - # https://rich.readthedocs.io/en/stable/reference/logging.html?highlight=rich%20handler#rich.logging.RichHandler - # Support compatible arguments between RichHandler and rich.traceback.install + # Rich traceback handling does not work on databricks. Hopefully this will be + # fixed on their side at some point, but until then we disable it. + # See https://github.com/Textualize/rich/issues/2455 rich.traceback.install(**traceback_install_kwargs) From 9d65959657eb670eaa4b9ea6c2a9a8c72c5fdc7c Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Wed, 24 May 2023 18:20:58 +0100 Subject: [PATCH 24/41] Fix tests Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 4 ++-- kedro/logging.py | 3 --- tests/framework/project/test_logging.py | 11 +++++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 7edf35741d..97134eb12e 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -19,7 +19,7 @@ We now give some common examples of how you might like to change your project's ### Using `KEDRO_LOGGING_CONFIG` environment variable -`KEDRO_LOGGING_CONFIG` is an optional environment variable that you can use to specify the path of your logging configuration file, overriding the default Kedro's framework `default_logging.yml`. +`KEDRO_LOGGING_CONFIG` is an optional environment variable that you can use to specify the path of your logging configuration file, overriding the default Kedro's `default_logging.yml`. To use this environment variable, set it to the path of your desired logging configuration file before running any Kedro commands. For example, if you have a logging configuration file located at `/path/to/logging.yml`, you can set `KEDRO_LOGGING_CONFIG` as follows: @@ -45,7 +45,7 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi ### Customize the `rich` Handler -Kedro's `kedro.logging.RichHandler` is a subclass of [`rich.logging.RichHand[label](vscode-file://vscode-a section of the class. +Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHand[label](vscode-file://vscode-a section of the class. pp/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html)ler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. diff --git a/kedro/logging.py b/kedro/logging.py index b8d665fc3c..a86456141f 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -52,9 +52,6 @@ def __init__(self, *args, **kwargs): traceback_install_kwargs[key_prefix_removed].extend(value) else: traceback_install_kwargs[key_prefix_removed] = value - # These two arguments are also compatible with RichHandler - elif key in ("locals_max_length", "locals_max_string"): - traceback_install_kwargs[key] = value if self.rich_tracebacks and "DATABRICKS_RUNTIME_VERSION" not in os.environ: # Rich traceback handling does not work on databricks. Hopefully this will be diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 4e426aa917..d44457739f 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -82,24 +82,27 @@ def test_rich_traceback_not_installed(mocker): def test_rich_traceback_configuration(mocker): rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") + import click sys_executable_path = str(Path(sys.executable).parent) traceback_install_defaults = {"suppress": [click, sys_executable_path]} - + fake_path = "dummy" rich_handler = { "class": "kedro.logging.RichHandler", "rich_tracebacks": True, "tracebacks_show_locals": True, + "tracebacks_suppress": [fake_path], } test_logging_config = default_logging_config.copy() test_logging_config["handlers"]["rich"] = rich_handler LOGGING.configure(test_logging_config) - rich_traceback_install.assert_called_with( - **{"show_locals": True, **traceback_install_defaults} - ) + expected_install_defaults = traceback_install_defaults.copy() + expected_install_defaults["suppress"].extend([fake_path]) + expected_install_defaults["show_locals"] = True + rich_traceback_install.assert_called_with(**expected_install_defaults) rich_pretty_install.assert_called_once() From 8b4f63a60a59ce829a0afa624c08458e4ffa973f Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 13:29:11 +0100 Subject: [PATCH 25/41] Link options to doc Signed-off-by: Nok Chan --- .../{{ cookiecutter.repo_name }}/conf/base/logging.yml | 4 +++- kedro/framework/project/default_logging.yml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml index 68e9762299..b3fc52c28a 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -36,13 +36,15 @@ handlers: rich: class: kedro.logging.RichHandler rich_tracebacks: True + # Advance options for customisation. + # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration # tracebacks_show_locals: False loggers: kedro: level: INFO - {{ cookiecutter.python_package }}: + { { cookiecutter.python_package } }: level: INFO root: diff --git a/kedro/framework/project/default_logging.yml b/kedro/framework/project/default_logging.yml index a89e348087..87fae8a25c 100644 --- a/kedro/framework/project/default_logging.yml +++ b/kedro/framework/project/default_logging.yml @@ -6,6 +6,8 @@ handlers: rich: class: kedro.logging.RichHandler rich_tracebacks: True + # Advance options for customisation. + # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration # tracebacks_show_locals: False loggers: From 0c166ad5e17b79dc248c02cd20e64abba28be8c7 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 13:33:33 +0100 Subject: [PATCH 26/41] Clean up doc and unused logger Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 6 ++---- kedro/logging.py | 5 +---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 97134eb12e..2575979ee3 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -30,7 +30,7 @@ export KEDRO_LOGGING_CONFIG=/path/to/logging.yml After setting the environment variable, any subsequent Kedro commands will use the logging configuration file at the specified path. ```{note} -If the `KEDRO_LOGGING_CONFIG` environment variable is not set, Kedro will default to using the logging configuration file at the project's default location of `conf/base/logging.yml`. +If the `KEDRO_LOGGING_CONFIG` environment variable is not set, Kedro will default to using the logging configuration file at the project's default location of Kedro's `default_logging.yml`. ``` ### Disable file-based logging @@ -45,9 +45,7 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi ### Customize the `rich` Handler -Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHand[label](vscode-file://vscode-a section of the class. - -pp/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-sandbox/workbench/workbench.html)ler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. +Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.traceback.install`](https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install). If an argument is compatible with `rich.traceback.install`, it will be passed to the traceback's settings. diff --git a/kedro/logging.py b/kedro/logging.py index a86456141f..03af52c64f 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -12,15 +12,12 @@ import rich.pretty import rich.traceback -logger = logging.getLogger(__file__) - - class RichHandler(rich.logging.RichHandler): """Identical to rich's logging handler but with a few extra behaviours: * warnings issued by the `warnings` module are redirected to logging * pretty printing is enabled on the Python REPL (including IPython and Jupyter) * all tracebacks are handled by rich when rich_tracebacks=True - * constructor's arguments are mapped and pass to `rich.traceback.install` + * constructor's arguments are mapped and passed to `rich.traceback.install` The list of available options of ``RichHandler`` can be found here: https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler From b81a5eab34287d4d4c365dd47288433f6ee622e7 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 13:35:11 +0100 Subject: [PATCH 27/41] Fix spelling `customise` Signed-off-by: Nok Chan --- docs/source/conf.py | 2 +- docs/source/logging/logging.md | 2 +- kedro/logging.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 58175c2c8b..7aa16b79e5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -191,7 +191,7 @@ here = Path(__file__).parent.absolute() html_logo = str(here / "kedro_logo.svg") -# Theme options are theme-specific and customize the look and feel of a theme +# Theme options are theme-specific and customise the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 2575979ee3..5ad33c35d2 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -43,7 +43,7 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi + handlers: [console] ``` -### Customize the `rich` Handler +### Customise the `rich` Handler Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. diff --git a/kedro/logging.py b/kedro/logging.py index 03af52c64f..cfe76d1506 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -40,7 +40,7 @@ def __init__(self, *args, **kwargs): "suppress": [click, str(Path(sys.executable).parent)] } - # Mapping Arguments from RichHandler's Constructor to rich.traceback.install + # Mapping arguments from RichHandler's Constructor to rich.traceback.install prefix = "tracebacks_" for key, value in kwargs.items(): if key.startswith(prefix): From 8f2788a7e69349bdfe29ed64848b887006e8007b Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 13:44:40 +0100 Subject: [PATCH 28/41] Refactor test with fixture Signed-off-by: Nok Chan --- tests/framework/project/test_logging.py | 36 ++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index d44457739f..4407c3f0d0 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -8,24 +8,28 @@ from kedro.framework.project import LOGGING, configure_logging -default_logging_config = { - "version": 1, - "disable_existing_loggers": False, - "handlers": { - "rich": {"class": "kedro.logging.RichHandler", "rich_tracebacks": True} - }, - "loggers": {"kedro": {"level": "INFO"}}, - "root": {"handlers": ["rich"]}, -} + +@pytest.fixture +def default_logging_config(): + logging_config = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "rich": {"class": "kedro.logging.RichHandler", "rich_tracebacks": True} + }, + "loggers": {"kedro": {"level": "INFO"}}, + "root": {"handlers": ["rich"]}, + } + return logging_config @pytest.fixture(autouse=True) -def reset_logging(): +def reset_logging(default_logging_config): yield configure_logging(default_logging_config) -def test_default_logging_config(): +def test_default_logging_config(default_logging_config): assert LOGGING.data == default_logging_config assert "rich" in {handler.name for handler in logging.getLogger().handlers} assert logging.getLogger("kedro").level == logging.INFO @@ -52,7 +56,7 @@ def test_configure_logging(): assert logging.getLogger("kedro").level == logging.WARNING -def test_rich_traceback_enabled(mocker): +def test_rich_traceback_enabled(mocker, default_logging_config): rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") @@ -62,7 +66,7 @@ def test_rich_traceback_enabled(mocker): rich_pretty_install.assert_called() -def test_rich_traceback_not_installed(mocker): +def test_rich_traceback_not_installed(mocker, default_logging_config): rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") rich_handler = { @@ -79,7 +83,7 @@ def test_rich_traceback_not_installed(mocker): rich_traceback_install.assert_not_called() -def test_rich_traceback_configuration(mocker): +def test_rich_traceback_configuration(mocker, default_logging_config): rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") @@ -106,7 +110,9 @@ def test_rich_traceback_configuration(mocker): rich_pretty_install.assert_called_once() -def test_rich_traceback_disabled_on_databricks(mocker, monkeypatch): +def test_rich_traceback_disabled_on_databricks( + mocker, monkeypatch, default_logging_config +): monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "1") rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") From f263931fb17fd4c7b82a12b3594a7e5016ee565f Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 13:46:17 +0100 Subject: [PATCH 29/41] Remove unnecessary copy() because it is now a fixture Signed-off-by: Nok Chan --- tests/framework/project/test_logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index 4407c3f0d0..f1e4c6cc5d 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -74,7 +74,7 @@ def test_rich_traceback_not_installed(mocker, default_logging_config): "rich_tracebacks": False, "tracebacks_show_locals": True, } - test_logging_config = default_logging_config.copy() + test_logging_config = default_logging_config test_logging_config["handlers"]["rich"] = rich_handler LOGGING.configure(test_logging_config) @@ -99,11 +99,11 @@ def test_rich_traceback_configuration(mocker, default_logging_config): "tracebacks_suppress": [fake_path], } - test_logging_config = default_logging_config.copy() + test_logging_config = default_logging_config test_logging_config["handlers"]["rich"] = rich_handler LOGGING.configure(test_logging_config) - expected_install_defaults = traceback_install_defaults.copy() + expected_install_defaults = traceback_install_defaults expected_install_defaults["suppress"].extend([fake_path]) expected_install_defaults["show_locals"] = True rich_traceback_install.assert_called_with(**expected_install_defaults) From 5071f5a9677e8c4a4f03f4d750fd2c8407602fbc Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 13:56:59 +0100 Subject: [PATCH 30/41] Split up test_rich_traceback_configuration() to smaller tests Signed-off-by: Nok Chan --- tests/framework/project/test_logging.py | 29 ++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/framework/project/test_logging.py b/tests/framework/project/test_logging.py index f1e4c6cc5d..52e7d5b4c1 100644 --- a/tests/framework/project/test_logging.py +++ b/tests/framework/project/test_logging.py @@ -72,7 +72,6 @@ def test_rich_traceback_not_installed(mocker, default_logging_config): rich_handler = { "class": "kedro.logging.RichHandler", "rich_tracebacks": False, - "tracebacks_show_locals": True, } test_logging_config = default_logging_config test_logging_config["handlers"]["rich"] = rich_handler @@ -84,18 +83,43 @@ def test_rich_traceback_not_installed(mocker, default_logging_config): def test_rich_traceback_configuration(mocker, default_logging_config): + import click + rich_traceback_install = mocker.patch("rich.traceback.install") rich_pretty_install = mocker.patch("rich.pretty.install") + sys_executable_path = str(Path(sys.executable).parent) + traceback_install_defaults = {"suppress": [click, sys_executable_path]} + + rich_handler = { + "class": "kedro.logging.RichHandler", + "rich_tracebacks": True, + "tracebacks_show_locals": True, + } + + test_logging_config = default_logging_config + test_logging_config["handlers"]["rich"] = rich_handler + LOGGING.configure(test_logging_config) + + expected_install_defaults = traceback_install_defaults + expected_install_defaults["show_locals"] = True + rich_traceback_install.assert_called_with(**expected_install_defaults) + rich_pretty_install.assert_called_once() + + +def test_rich_traceback_configuration_extend_suppress(mocker, default_logging_config): + """Test the configuration is not overrided but extend for `suppress`""" import click + rich_traceback_install = mocker.patch("rich.traceback.install") + rich_pretty_install = mocker.patch("rich.pretty.install") + sys_executable_path = str(Path(sys.executable).parent) traceback_install_defaults = {"suppress": [click, sys_executable_path]} fake_path = "dummy" rich_handler = { "class": "kedro.logging.RichHandler", "rich_tracebacks": True, - "tracebacks_show_locals": True, "tracebacks_suppress": [fake_path], } @@ -105,7 +129,6 @@ def test_rich_traceback_configuration(mocker, default_logging_config): expected_install_defaults = traceback_install_defaults expected_install_defaults["suppress"].extend([fake_path]) - expected_install_defaults["show_locals"] = True rich_traceback_install.assert_called_with(**expected_install_defaults) rich_pretty_install.assert_called_once() From f9d57696e55fc9e0f718cb6d583bbff8c7e7bb1e Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 14:01:06 +0100 Subject: [PATCH 31/41] Add RELEASE.md Signed-off-by: Nok Chan --- RELEASE.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index e71eb13406..6afc8e11a6 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -13,6 +13,7 @@ ## Major features and improvements * `kedro run --params` now updates interpolated parameters correctly when using `OmegaConfigLoader`. * Added `metadata` attribute to `kedro.io` datasets. This is ignored by Kedro, but may be consumed by users or external plugins. +* Added `kedro.logging.RichHandler`. This replaces the default `rich.logging.RichHandler` and is more flexible, user can turn off the `rich` traceback if needed. ## Bug fixes and other changes * `OmegaConfigLoader` will return a `dict` instead of `DictConfig`. @@ -140,12 +141,12 @@ Many thanks to the following Kedroids for contributing PRs to this release: * You can configure config file patterns through `settings.py` without creating a custom config loader. * Added the following new datasets: -| Type | Description | Location | -| ------------------------------------ | -------------------------------------------------------------------------- | ----------------------------- | -| `svmlight.SVMLightDataSet` | Work with svmlight/libsvm files using scikit-learn library | `kedro.extras.datasets.svmlight` | -| `video.VideoDataSet` | Read and write video files from a filesystem | `kedro.extras.datasets.video` | -| `video.video_dataset.SequenceVideo` | Create a video object from an iterable sequence to use with `VideoDataSet` | `kedro.extras.datasets.video` | -| `video.video_dataset.GeneratorVideo` | Create a video object from a generator to use with `VideoDataSet` | `kedro.extras.datasets.video` | +| Type | Description | Location | +| ------------------------------------ | -------------------------------------------------------------------------- | -------------------------------- | +| `svmlight.SVMLightDataSet` | Work with svmlight/libsvm files using scikit-learn library | `kedro.extras.datasets.svmlight` | +| `video.VideoDataSet` | Read and write video files from a filesystem | `kedro.extras.datasets.video` | +| `video.video_dataset.SequenceVideo` | Create a video object from an iterable sequence to use with `VideoDataSet` | `kedro.extras.datasets.video` | +| `video.video_dataset.GeneratorVideo` | Create a video object from a generator to use with `VideoDataSet` | `kedro.extras.datasets.video` | * Implemented support for a functional definition of schema in `dask.ParquetDataSet` to work with the `dask.to_parquet` API. ## Bug fixes and other changes From df58139e570962240c2c4b475927abe21b6c964b Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 17:10:55 +0100 Subject: [PATCH 32/41] Update doc Signed-off-by: Nok Chan --- docs/source/logging/logging.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/logging/logging.md b/docs/source/logging/logging.md index 5ad33c35d2..93f3332660 100644 --- a/docs/source/logging/logging.md +++ b/docs/source/logging/logging.md @@ -47,6 +47,10 @@ Alternatively, if you would like to keep other configuration in `conf/base/loggi Kedro's `kedro.extras.logging.RichHandler` is a subclass of [`rich.logging.RichHandler`](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler) and supports the same set of arguments. By default, `rich_tracebacks` is set to `True` to use `rich` to render exceptions. However, you can disable it by setting `rich_tracebacks: False`. +```{note} +If you want to disable `rich`'s tracebacks, you must set `KEDRO_LOGGING_CONFIG` to point to your local config i.e. `conf/base/logging.yml`. +``` + When `rich_tracebacks` is set to `True`, the configuration is propagated to [`rich.traceback.install`](https://rich.readthedocs.io/en/stable/reference/traceback.html#rich.traceback.install). If an argument is compatible with `rich.traceback.install`, it will be passed to the traceback's settings. For instance, you can enable the display of local variables inside `logging.yml` to aid with debugging. @@ -60,6 +64,7 @@ rich: A comprehensive list of available options can be found in the [RichHandler documentation](https://rich.readthedocs.io/en/stable/reference/logging.html#rich.logging.RichHandler). + ### Use plain console logging To use plain rather than rich logging, swap the `rich` handler for the `console` one as follows: From af4a9ce227387d5c106d77dff97daa55d6509d7f Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Thu, 25 May 2023 17:54:25 +0100 Subject: [PATCH 33/41] Fix template logging Signed-off-by: Nok Chan --- .../{{ cookiecutter.repo_name }}/conf/base/logging.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml index 137f193c18..c6a6fc7057 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -26,7 +26,9 @@ handlers: rich: class: kedro.logging.RichHandler rich_tracebacks: True - # tracebacks_show_locals: True + # Advance options for customisation. + # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration + # tracebacks_show_locals: False loggers: kedro: From d123a3e82643c241602edec40138cd45e9ba4555 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 26 May 2023 15:35:55 +0100 Subject: [PATCH 34/41] Fix the. template Signed-off-by: Nok Chan --- .../{{ cookiecutter.repo_name }}/conf/base/logging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml index b3fc52c28a..d60b7d3592 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml @@ -44,7 +44,7 @@ loggers: kedro: level: INFO - { { cookiecutter.python_package } }: + {{ cookiecutter.python_package }}: level: INFO root: From 6ad446327588b1aced57aa8af76d9a7ca1cd0151 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Fri, 26 May 2023 15:52:34 +0100 Subject: [PATCH 35/41] Lint Signed-off-by: Nok Chan --- kedro/logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kedro/logging.py b/kedro/logging.py index cfe76d1506..534776c566 100644 --- a/kedro/logging.py +++ b/kedro/logging.py @@ -12,6 +12,7 @@ import rich.pretty import rich.traceback + class RichHandler(rich.logging.RichHandler): """Identical to rich's logging handler but with a few extra behaviours: * warnings issued by the `warnings` module are redirected to logging From b883906b58d974e830504b11b77b03c6e153dfca Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 30 May 2023 12:40:22 +0100 Subject: [PATCH 36/41] Add doc to fix doc issues Signed-off-by: Nok Chan --- docs/source/kedro.logging.RichHandler.rst | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/source/kedro.logging.RichHandler.rst diff --git a/docs/source/kedro.logging.RichHandler.rst b/docs/source/kedro.logging.RichHandler.rst new file mode 100644 index 0000000000..fb7479dd3c --- /dev/null +++ b/docs/source/kedro.logging.RichHandler.rst @@ -0,0 +1,26 @@ +kedro.logging.RichHandler +========================= + +.. currentmodule:: kedro.logging + +.. autoclass:: RichHandler + :members: + :undoc-members: + :inherited-members: + + + + .. rubric:: Attributes + + .. autosummary:: + + + + + + + .. rubric:: Methods + + .. autosummary:: + + From e749dca27245d47fc6d8bb2a68bbdfb5ceda6771 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 30 May 2023 13:07:20 +0100 Subject: [PATCH 37/41] fix doc Signed-off-by: Nok Chan --- docs/source/kedro.logging.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/source/kedro.logging.rst diff --git a/docs/source/kedro.logging.rst b/docs/source/kedro.logging.rst new file mode 100644 index 0000000000..c05b1a1c40 --- /dev/null +++ b/docs/source/kedro.logging.rst @@ -0,0 +1,29 @@ +kedro.logging +============= + +.. rubric:: Description + +.. automodule:: kedro.logging + + + + + + + + .. rubric:: Classes + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + + RichHandler + + + + + + + + + From c726d6e9b723069d24db6f8541779543ba86aa07 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 30 May 2023 15:03:10 +0100 Subject: [PATCH 38/41] Fix doc rst so it don't reference to the upstream class Signed-off-by: Nok Chan --- docs/source/kedro.logging.RichHandler.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/source/kedro.logging.RichHandler.rst b/docs/source/kedro.logging.RichHandler.rst index fb7479dd3c..9bd6cbd71a 100644 --- a/docs/source/kedro.logging.RichHandler.rst +++ b/docs/source/kedro.logging.RichHandler.rst @@ -9,18 +9,3 @@ kedro.logging.RichHandler :inherited-members: - - .. rubric:: Attributes - - .. autosummary:: - - - - - - - .. rubric:: Methods - - .. autosummary:: - - From af3e3d0bec0c5933da10d5a7b2f9b1017163ca1f Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 30 May 2023 15:03:42 +0100 Subject: [PATCH 39/41] Remvoe whitespace Signed-off-by: Nok Chan --- docs/source/kedro.logging.RichHandler.rst | 2 -- docs/source/kedro.logging.rst | 21 ++++++--------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/docs/source/kedro.logging.RichHandler.rst b/docs/source/kedro.logging.RichHandler.rst index 9bd6cbd71a..11e747b111 100644 --- a/docs/source/kedro.logging.RichHandler.rst +++ b/docs/source/kedro.logging.RichHandler.rst @@ -7,5 +7,3 @@ kedro.logging.RichHandler :members: :undoc-members: :inherited-members: - - diff --git a/docs/source/kedro.logging.rst b/docs/source/kedro.logging.rst index c05b1a1c40..9476656b50 100644 --- a/docs/source/kedro.logging.rst +++ b/docs/source/kedro.logging.rst @@ -5,25 +5,16 @@ kedro.logging .. automodule:: kedro.logging - - - - - + + + + + .. rubric:: Classes .. autosummary:: :toctree: :template: autosummary/class.rst - - RichHandler - - - - - - - - + RichHandler From 40e91561c408bcafcfa3d31361ae0d7b7d2c23e9 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 30 May 2023 15:19:24 +0100 Subject: [PATCH 40/41] remove undoc member and inherited-member for RichHandler Signed-off-by: Nok Chan --- docs/source/kedro.logging.RichHandler.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/kedro.logging.RichHandler.rst b/docs/source/kedro.logging.RichHandler.rst index 11e747b111..40a9e7f841 100644 --- a/docs/source/kedro.logging.RichHandler.rst +++ b/docs/source/kedro.logging.RichHandler.rst @@ -4,6 +4,4 @@ kedro.logging.RichHandler .. currentmodule:: kedro.logging .. autoclass:: RichHandler - :members: - :undoc-members: - :inherited-members: + From b7d94e6adbaa5598ad06243294b649baf5265104 Mon Sep 17 00:00:00 2001 From: Nok Chan Date: Tue, 30 May 2023 15:34:51 +0100 Subject: [PATCH 41/41] Please the linter once again Signed-off-by: Nok Chan --- docs/source/kedro.logging.RichHandler.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/kedro.logging.RichHandler.rst b/docs/source/kedro.logging.RichHandler.rst index 40a9e7f841..b14b9e91aa 100644 --- a/docs/source/kedro.logging.RichHandler.rst +++ b/docs/source/kedro.logging.RichHandler.rst @@ -4,4 +4,3 @@ kedro.logging.RichHandler .. currentmodule:: kedro.logging .. autoclass:: RichHandler -