diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c9597d043..60c45abdc1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,7 @@ repos: - types-requests - types-setuptools - types-toml + - attrs - repo: https://github.com/asottile/blacken-docs rev: v1.12.1 diff --git a/RELEASE.md b/RELEASE.md index a28387e1a1..174a8d2935 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,6 +18,7 @@ * Added `abfss` to list of cloud protocols, enabling abfss paths. * Kedro now uses the [Rich](https://github.com/Textualize/rich) library to format terminal logs and tracebacks. * The file `conf/base/logging.yml` is now optional. See [our documentation](https://kedro.readthedocs.io/en/0.18.2/logging/logging.html) for details. +* Introduced a `kedro.starters` entry point. This enables plugins to create custom starter aliases used by `kedro starter list` and `kedro new`. ## Bug fixes and other changes * Bumped `pyyaml` upper bound to make Kedro compatible with the [pyodide](https://pyodide.org/en/stable/usage/loading-packages.html#micropip) stack. @@ -34,6 +35,7 @@ ## Minor breaking changes to the API * The module `kedro.config.default_logger` no longer exists; default logging configuration is now set automatically through `kedro.framework.project.LOGGING`. Unless you explicitly import `kedro.config.default_logger` you do not need to make any changes. + ## Upcoming deprecations for Kedro 0.19.0 * `kedro.extras.ColorHandler` will be removed in 0.19.0. diff --git a/docs/source/extend_kedro/create_kedro_starters.md b/docs/source/extend_kedro/create_kedro_starters.md index a91958cbca..d3bd783829 100644 --- a/docs/source/extend_kedro/create_kedro_starters.md +++ b/docs/source/extend_kedro/create_kedro_starters.md @@ -87,3 +87,7 @@ Here is the layout of the project as a Cookiecutter template: ├── setup.py └── tests ``` + +```{note} +You can [add an alias by creating a plugin using `kedro.starters` entry point](./plugins.md#extend-starter-aliases), which will allows you to do `kedro new --starter=your_starters` and shows up on shows up on `kedro starter list`. +``` diff --git a/docs/source/extend_kedro/plugins.md b/docs/source/extend_kedro/plugins.md index d9272fbd4b..bbf4eaf8d9 100644 --- a/docs/source/extend_kedro/plugins.md +++ b/docs/source/extend_kedro/plugins.md @@ -44,6 +44,31 @@ Once the plugin is installed, you can run it as follows: kedro to_json ``` +## Extend starter aliases +It is possible to extend the list of starter aliases built into Kedro. This means that a [custom Kedro starter](create_kedro_starters.md) can be used directly through the `starter` argument in `kedro new` rather than needing to explicitly provide the `template` and `directory` arguments. A custom starter alias behaves in the same way as an official Kedro starter alias and is also picked up by `kedro starter list`. + +You need to extend the starters by providing a list of `KedroStarterSpec`, in this example it is defined in a file called `plugin.py`. + +```python +starters = [ + KedroStarterSpec( + alias="test_plugin_starter", + template_path="https://github.com/kedro-org/kedro-starters/", + directory="pandas-iris", + ) +] +``` + +The `directory` argument is optional and should be used when you have multiple templates in one repository as for the [official kedro-starters](https://github.com/kedro-org/kedro-starters). If you only have one template, your top-level directory will be treated as the template. You can take the [pandas-iris](https://github.com/kedro-org/kedro-starters/tree/main/pandas-iris) example for reference. + +In your `setup.py`, you need to register the specifications to `kedro.starters`. + +```python +setup( + entry_points={"kedro.starters": ["starter = plugin:starters"]}, +) +``` + ## Working with `click` Commands must be provided as [`click` `Groups`](https://click.palletsprojects.com/en/7.x/api/#click.Group) diff --git a/features/activate_nbstripout.feature b/features/activate_nbstripout.feature index 78a7bafd65..fa221417d4 100644 --- a/features/activate_nbstripout.feature +++ b/features/activate_nbstripout.feature @@ -2,7 +2,7 @@ Feature: Activate_nbstripout target in new project Scenario: Check nbstripout git post commit hook functionality Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" And I have added a test jupyter notebook And I have initialized a git repository And I have added the project directory to staging diff --git a/features/build_docs.feature b/features/build_docs.feature index 1797a06e14..c9f9307ef1 100644 --- a/features/build_docs.feature +++ b/features/build_docs.feature @@ -3,7 +3,7 @@ Feature: build-docs target in new project @fresh_venv Scenario: Execute build-docs target Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" And I have updated kedro requirements And I have installed the project dependencies When I execute the kedro command "build-docs" diff --git a/features/build_reqs.feature b/features/build_reqs.feature index 8630f3f2ba..085cab2242 100644 --- a/features/build_reqs.feature +++ b/features/build_reqs.feature @@ -3,7 +3,7 @@ Feature: build-reqs target in new project @fresh_venv Scenario: Execute build-reqs target Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" And I have updated kedro requirements And I have executed the kedro command "build-reqs" When I add scrapy>=1.7.3 to the requirements diff --git a/features/info.feature b/features/info.feature index 6353bea64e..a4adc3eab3 100644 --- a/features/info.feature +++ b/features/info.feature @@ -1,7 +1,7 @@ Feature: Run kedro info Background: Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" Scenario: Plugins are installed and detected by kedro info Given I have installed the test plugin diff --git a/features/ipython.feature b/features/ipython.feature index 8480bf52b4..c42dd5daaa 100644 --- a/features/ipython.feature +++ b/features/ipython.feature @@ -2,7 +2,7 @@ Feature: IPython target in new project Scenario: Execute ipython target Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" When I execute the kedro command "ipython" Then I should get a message including "An enhanced Interactive Python" And I should get a message including "Kedro project project-dummy" diff --git a/features/jupyter.feature b/features/jupyter.feature index baf938714c..e57ee540b3 100644 --- a/features/jupyter.feature +++ b/features/jupyter.feature @@ -2,7 +2,7 @@ Feature: Jupyter targets in new project Background: Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" Scenario: Execute jupyter-notebook target When I execute the kedro jupyter command "notebook --no-browser" diff --git a/features/load_context.feature b/features/load_context.feature index 8e3d356386..7930cf2b8b 100644 --- a/features/load_context.feature +++ b/features/load_context.feature @@ -1,7 +1,7 @@ Feature: Custom Kedro project Background: Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" Scenario: Update the source directory to be nested When I move the package to "src/nested" diff --git a/features/new_project.feature b/features/new_project.feature index f99ee7135c..3889384657 100644 --- a/features/new_project.feature +++ b/features/new_project.feature @@ -1,13 +1,19 @@ Feature: New Kedro project + Background: + Given I have prepared a config file Scenario: Create a new kedro project without example code - Given I have prepared a config file When I run a non-interactive kedro new without starter Then the expected project directories and files should be created And the pipeline should contain no nodes Scenario: Create a new kedro project with example code - Given I have prepared a config file - When I run a non-interactive kedro new with starter + When I run a non-interactive kedro new with starter "default" + Then the expected project directories and files should be created + And the pipeline should contain nodes + + Scenario: Plugins are installed and create a new kedro project with custom plugin starter + Given I have installed the test plugin + When I run a non-interactive kedro new with starter "test_plugin_starter" Then the expected project directories and files should be created And the pipeline should contain nodes diff --git a/features/package.feature b/features/package.feature index 638c98de07..21873e775c 100644 --- a/features/package.feature +++ b/features/package.feature @@ -2,7 +2,7 @@ Feature: Package target in new project Background: Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" And I have installed the project dependencies @fresh_venv diff --git a/features/run.feature b/features/run.feature index f49fb0171f..3ee93c259a 100644 --- a/features/run.feature +++ b/features/run.feature @@ -6,14 +6,14 @@ Feature: Run Project Local environment should be used by default when no env option is specified. Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" When I execute the kedro command "run" Then I should get a successful exit code And the logs should show that 4 nodes were run Scenario: Run parallel runner with default python entry point with example code Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" When I execute the kedro command "run --runner=ParallelRunner" Then I should get a successful exit code And the logs should show that "split_data" was run @@ -30,7 +30,7 @@ Feature: Run Project Scenario: Run kedro run with config file Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" And I have prepared a run_config file with config options When I execute the kedro command "run --config run_config.yml" Then I should get a successful exit code @@ -38,7 +38,7 @@ Feature: Run Project Scenario: Run kedro run with config file and override option Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" And I have prepared a run_config file with config options When I execute the kedro command "run --config run_config.yml --pipeline __default__" Then I should get a successful exit code @@ -46,7 +46,7 @@ Feature: Run Project Scenario: Run kedro run with extra parameters Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" When I execute the kedro command "run --params extra1:1,extra2:value2" Then I should get a successful exit code And the logs should show that 4 nodes were run diff --git a/features/starter.feature b/features/starter.feature new file mode 100644 index 0000000000..2449324846 --- /dev/null +++ b/features/starter.feature @@ -0,0 +1,9 @@ +Feature: List Kedro Starters + + Scenario: List all starters with custom starters from plugin + Given I have prepared a config file + And I have installed the test plugin + And I have run a non-interactive kedro new with starter "default" + When I execute the kedro command "starter list" + Then I should get a successful exit code + And I should get a message including "test_plugin_starter" diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 7621a7461c..f244631f17 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -222,11 +222,13 @@ def add_test_jupyter_nb(context): test_nb_fh.write(TEST_JUPYTER_ORG) -@given("I have run a non-interactive kedro new with starter") -@when("I run a non-interactive kedro new with starter") -def create_project_with_starter(context): +@given('I have run a non-interactive kedro new with starter "{starter}"') +@when('I run a non-interactive kedro new with starter "{starter}"') +def create_project_with_starter(context, starter): """Behave step to run kedro new given the config I previously created.""" - starter_dir = Path(__file__).parent / "test_starter" + if starter == "default": + starter = Path(__file__).parent / "test_starter" + res = run( [ context.kedro, @@ -234,7 +236,7 @@ def create_project_with_starter(context): "-c", str(context.config_file), "--starter", - str(starter_dir), + str(starter), ], env=context.env, cwd=context.temp_dir, @@ -468,7 +470,7 @@ def check_correct_nodes_run(context, node): "Expected the following message segment to be printed on stdout: " f"{expected_log_line},\nbut got {stdout}" ) - assert expected_log_line in info_log.read_text() + assert expected_log_line in info_log.read_text(), info_log.read_text() @then("I should get a successful exit code") diff --git a/features/steps/test_plugin/plugin.py b/features/steps/test_plugin/plugin.py index 82b3e73832..4e699195da 100644 --- a/features/steps/test_plugin/plugin.py +++ b/features/steps/test_plugin/plugin.py @@ -1,6 +1,7 @@ """Dummy plugin with simple hook implementations.""" import logging +from kedro.framework.cli.starters import KedroStarterSpec from kedro.framework.hooks import hook_impl logger = logging.getLogger(__name__) @@ -11,8 +12,16 @@ class MyPluginHook: @hook_impl def after_catalog_created( self, catalog - ): # pylint: disable=unused-argument,no-self-use + ): # pylint: disable=unused-argument, no-self-use logger.info("Reached after_catalog_created hook") +starters = [ + KedroStarterSpec( + "test_plugin_starter", + template_path="https://github.com/kedro-org/kedro-starters/", + directory="pandas-iris", + ) +] + hooks = MyPluginHook() diff --git a/features/steps/test_plugin/setup.py b/features/steps/test_plugin/setup.py index 0766248c39..d76b760a78 100644 --- a/features/steps/test_plugin/setup.py +++ b/features/steps/test_plugin/setup.py @@ -3,7 +3,10 @@ setup( name="test_plugin", version="0.1", - description="Dummy plugin with hook implementations", + description="Dummy plugin with hook implementations and custom starters", packages=find_packages(), - entry_points={"kedro.hooks": ["test_plugin = plugin:hooks"]}, + entry_points={ + "kedro.hooks": ["test_plugin = plugin:hooks"], + "kedro.starters": ["starter = plugin:starters"], + }, ) diff --git a/features/test.feature b/features/test.feature index 2b5235adb0..0d42f336e6 100644 --- a/features/test.feature +++ b/features/test.feature @@ -2,7 +2,7 @@ Feature: Test target in new project Background: Given I have prepared a config file - And I have run a non-interactive kedro new with starter + And I have run a non-interactive kedro new with starter "default" Scenario: Execute successful test in new project When I execute the kedro command "test" diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index d2e5dfa8eb..e8be72eab7 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -10,7 +10,6 @@ from typing import Sequence import click -import importlib_metadata from kedro import __version__ as version from kedro.framework.cli.catalog import catalog_cli @@ -26,6 +25,7 @@ ENTRY_POINT_GROUPS, CommandCollection, KedroCliError, + _get_entry_points, load_entry_points, ) from kedro.framework.project import LOGGING # noqa # pylint:disable=unused-import @@ -63,8 +63,8 @@ def info(): plugin_versions = {} plugin_entry_points = defaultdict(set) - for plugin_entry_point, group in ENTRY_POINT_GROUPS.items(): - for entry_point in importlib_metadata.entry_points().select(group=group): + for plugin_entry_point in ENTRY_POINT_GROUPS: + for entry_point in _get_entry_points(plugin_entry_point): module_name = entry_point.module.split(".")[0] plugin_versions[module_name] = entry_point.dist.version plugin_entry_points[module_name].add(plugin_entry_point) diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index f82ba9c99a..10eb480311 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -9,11 +9,13 @@ import stat import tempfile from collections import OrderedDict +from itertools import groupby from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple import click import yaml +from attrs import define, field import kedro from kedro import __version__ as version @@ -22,22 +24,55 @@ KedroCliError, _clean_pycache, _filter_deprecation_warnings, + _get_entry_points, + _safe_load_entry_point, command_with_verbosity, ) KEDRO_PATH = Path(kedro.__file__).parent TEMPLATE_PATH = KEDRO_PATH / "templates" / "project" - -_STARTER_ALIASES = { - "astro-airflow-iris", - "standalone-datacatalog", - "pandas-iris", - "pyspark", - "pyspark-iris", - "spaceflights", -} _STARTERS_REPO = "git+https://github.com/kedro-org/kedro-starters.git" + +@define(order=True) +class KedroStarterSpec: # pylint: disable=too-few-public-methods + """Specification of custom kedro starter template + Args: + alias: alias of the starter which shows up on `kedro starter list` and is used + by the starter argument of `kedro new` + template_path: path to a directory or a URL to a remote VCS repository supported + by `cookiecutter` + directory: optional directory inside the repository where the starter resides. + origin: reserved field used by kedro internally to determine where the starter + comes from, users do not need to provide this field. + """ + + alias: str + template_path: str + directory: Optional[str] = None + origin: Optional[str] = field(init=False) + + +_OFFICIAL_STARTER_SPECS = [ + KedroStarterSpec("astro-airflow-iris", _STARTERS_REPO, "astro-airflow-iris"), + # The `astro-iris` was renamed to `astro-airflow-iris`, but old (external) + # documentation and tutorials still refer to `astro-iris`. We create an alias to + # check if a user has entered old `astro-iris` as the starter name and changes it + # to `astro-airflow-iris`. + KedroStarterSpec("astro-iris", _STARTERS_REPO, "astro-airflow-iris"), + KedroStarterSpec( + "standalone-datacatalog", _STARTERS_REPO, "standalone-datacatalog" + ), + KedroStarterSpec("pyspark", _STARTERS_REPO, "pyspark"), + KedroStarterSpec("pyspark-iris", _STARTERS_REPO, "pyspark-iris"), + KedroStarterSpec("spaceflights", _STARTERS_REPO, "spaceflights"), +] +# Set the origin for official starters +for starter_spec in _OFFICIAL_STARTER_SPECS: + starter_spec.origin = "kedro" +_OFFICIAL_STARTER_SPECS = {spec.alias: spec for spec in _OFFICIAL_STARTER_SPECS} + + CONFIG_ARG_HELP = """Non-interactive mode, using a configuration yaml file. This file must supply the keys required by the template's prompts.yml. When not using a starter, these are `project_name`, `repo_name` and `python_package`.""" @@ -62,6 +97,65 @@ def _remove_readonly(func: Callable, path: Path, excinfo: Tuple): # pragma: no func(path) +def _get_starters_dict() -> Dict[str, KedroStarterSpec]: + """This function lists all the starter aliases declared in + the core repo and in plugins entry points. + + For example, the output for official kedro starters looks like: + {"astro-airflow-iris": + KedroStarterSpec( + name="astro-airflow-iris", + template_path="git+https://github.com/kedro-org/kedro-starters.git", + directory="astro-airflow-iris", + origin="kedro" + ), + "astro-iris": + KedroStarterSpec( + name="astro-iris", + template_path="git+https://github.com/kedro-org/kedro-starters.git", + directory="astro-airflow-iris", + origin="kedro" + ), + } + """ + starter_specs = _OFFICIAL_STARTER_SPECS + + for starter_entry_point in _get_entry_points(name="starters"): + origin = starter_entry_point.module.split(".")[0] + specs = _safe_load_entry_point(starter_entry_point) or [] + for spec in specs: + if not isinstance(spec, KedroStarterSpec): + click.secho( + f"The starter configuration loaded from module {origin}" + f"should be a 'KedroStarterSpec', got '{type(spec)}' instead", + fg="red", + ) + elif spec.alias in starter_specs: + click.secho( + f"Starter alias `{spec.alias}` from `{origin}` " + f"has been ignored as it is already defined by" + f"`{starter_specs[spec.alias].origin}`", + fg="red", + ) + else: + spec.origin = origin + starter_specs[spec.alias] = spec + return starter_specs + + +def _starter_spec_to_dict( + starter_specs: Dict[str, KedroStarterSpec] +) -> Dict[str, Dict[str, str]]: + """Convert a dictionary of starters spec to a nicely formatted dictionary""" + format_dict: Dict[str, Dict[str, str]] = {} + for alias, spec in starter_specs.items(): + format_dict[alias] = {} # Each dictionary represent 1 starter + format_dict[alias]["template_path"] = spec.template_path + if spec.directory: + format_dict[alias]["directory"] = spec.directory + return format_dict + + # pylint: disable=missing-function-docstring @click.group(context_settings=CONTEXT_SETTINGS, name="Kedro") def create_cli(): # pragma: no cover @@ -76,35 +170,34 @@ def create_cli(): # pragma: no cover type=click.Path(exists=True), help=CONFIG_ARG_HELP, ) -@click.option("--starter", "-s", "starter_name", help=STARTER_ARG_HELP) +@click.option("--starter", "-s", "starter_alias", help=STARTER_ARG_HELP) @click.option("--checkout", help=CHECKOUT_ARG_HELP) @click.option("--directory", help=DIRECTORY_ARG_HELP) -def new(config_path, starter_name, checkout, directory, **kwargs): +def new(config_path, starter_alias, checkout, directory, **kwargs): """Create a new kedro project.""" - if checkout and not starter_name: + if checkout and not starter_alias: raise KedroCliError("Cannot use the --checkout flag without a --starter value.") - if directory and not starter_name: + if directory and not starter_alias: raise KedroCliError( "Cannot use the --directory flag without a --starter value." ) - # The `astro-iris` was renamed to `astro-airflow-iris`, but old (external) documentation - # and tutorials still refer to `astro-iris`. The below line checks if a user has entered old - # `astro-iris` as the starter name and changes it to `astro-airflow-iris`. - starter_name = ( - "astro-airflow-iris" if starter_name == "astro-iris" else starter_name - ) - if starter_name in _STARTER_ALIASES: + starters_dict = _get_starters_dict() + + if starter_alias in starters_dict: if directory: raise KedroCliError( "Cannot use the --directory flag with a --starter alias." ) - template_path = _STARTERS_REPO - directory = starter_name + spec = starters_dict[starter_alias] + template_path = spec.template_path + # "directory" is an optional key for starters from plugins, so if the key is + # not present we will use "None". + directory = spec.directory checkout = checkout or version - elif starter_name is not None: - template_path = starter_name + elif starter_alias is not None: + template_path = starter_alias checkout = checkout or version else: template_path = str(TEMPLATE_PATH) @@ -147,11 +240,26 @@ def starter(): @starter.command("list") def list_starters(): """List all official project starters available.""" - repo_url = _STARTERS_REPO.replace("git+", "").replace(".git", "/tree/main/{alias}") - output = [ - {alias: repo_url.format(alias=alias)} for alias in sorted(_STARTER_ALIASES) - ] - click.echo(yaml.safe_dump(output)) + starters_dict = _get_starters_dict() + + # Group all specs by origin as nested dict and sort it. + sorted_starters_dict: Dict[str, Dict[str, KedroStarterSpec]] = { + origin: dict(sorted(starters_dict_by_origin)) + for origin, starters_dict_by_origin in groupby( + starters_dict.items(), lambda item: item[1].origin + ) + } + + # ensure kedro starters are listed first + sorted_starters_dict = dict( + sorted(sorted_starters_dict.items(), key=lambda x: x == "kedro") + ) + + for origin, starters_spec in sorted_starters_dict.items(): + click.secho(f"\nStarters from {origin}\n", fg="yellow") + click.echo( + yaml.safe_dump(_starter_spec_to_dict(starters_spec), sort_keys=False) + ) def _fetch_config_from_file(config_path: str) -> Dict[str, str]: @@ -286,11 +394,10 @@ def _get_cookiecutter_dir( f" Specified tag {checkout}. The following tags are available: " + ", ".join(_get_available_tags(template_path)) ) - official_starters = sorted(_STARTER_ALIASES) - + official_starters = sorted(_OFFICIAL_STARTER_SPECS) raise KedroCliError( f"{error_message}. The aliases for the official Kedro starters are: \n" - f"{yaml.safe_dump(official_starters)}" + f"{yaml.safe_dump(official_starters, sort_keys=False)}" ) from exc return Path(cookiecutter_dir) diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index 5131f1f3fc..76b092c146 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -32,6 +32,7 @@ "line_magic": "kedro.line_magic", "hooks": "kedro.hooks", "cli_hooks": "kedro.cli_hooks", + "starters": "kedro.starters", } logger = logging.getLogger(__name__) @@ -318,6 +319,27 @@ def _check_module_importable(module_name: str) -> None: ) from exc +def _get_entry_points(name: str) -> importlib_metadata.EntryPoints: + """Get all kedro related entry points""" + return importlib_metadata.entry_points().select(group=ENTRY_POINT_GROUPS[name]) + + +def _safe_load_entry_point( # pylint: disable=inconsistent-return-statements + entry_point, +): + """Load entrypoint safely, if fails it will just skip the entrypoint.""" + try: + return entry_point.load() + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "Failed to load %s commands from %s. Full exception: %s", + entry_point.module, + entry_point, + exc, + ) + return + + def load_entry_points(name: str) -> Sequence[click.MultiCommand]: """Load package entry point commands. @@ -331,20 +353,12 @@ def load_entry_points(name: str) -> Sequence[click.MultiCommand]: List of entry point commands. """ - entry_points = importlib_metadata.entry_points() - entry_points = entry_points.select(group=ENTRY_POINT_GROUPS[name]) entry_point_commands = [] - for entry_point in entry_points: - try: - entry_point_commands.append(entry_point.load()) - except Exception as exc: # pylint: disable=broad-except - logger.warning( - "Failed to load %s commands from %s. Full exception: %s", - name, - entry_point, - exc, - ) + for entry_point in _get_entry_points(name): + loaded_entry_point = _safe_load_entry_point(entry_point) + if loaded_entry_point: + entry_point_commands.append(loaded_entry_point) return entry_point_commands diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 22045743c8..06bb9e470f 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -206,7 +206,8 @@ def __init__(self): def configure(self, logging_config: Dict[str, Any]) -> None: """Configure project logging using `logging_config` (e.g. from project logging.yml). We store this in the UserDict data so that it can be reconfigured - in _bootstrap_subprocess.""" + in _bootstrap_subprocess. + """ logging.config.dictConfig(logging_config) self.data = logging_config diff --git a/pyproject.toml b/pyproject.toml index 3de08a9ce1..ff54547961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,8 @@ +# PEP-518 https://peps.python.org/pep-0518/ +[build-system] +# Minimum requirements for the build system to execute. +requires = ["setuptools>=38.0", "wheel"] # PEP 518 specifications. + [tool.black] exclude = "/templates/|^features/steps/test_starter" diff --git a/requirements.txt b/requirements.txt index 4958ae1505..e50d7d1c37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ anyconfig~=0.10.0 +attrs~=21.3 cachetools~=4.1 click<9.0 cookiecutter>=2.1.1, <3.0 diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index eb4a807e2f..91f82aa392 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -98,7 +98,7 @@ def test_info_contains_plugin_versions(self, entry_point): result = CliRunner().invoke(cli, ["info"]) assert result.exit_code == 0 assert ( - "bob: 1.0.2 (entry points:cli_hooks,global,hooks,init,line_magic,project)" + "bob: 1.0.2 (entry points:cli_hooks,global,hooks,init,line_magic,project,starters)" in result.output ) @@ -303,6 +303,7 @@ def test_project_groups(self, entry_points, entry_point): def test_project_error_is_caught(self, entry_points, entry_point, caplog): entry_point.load.side_effect = Exception() + entry_point.module = "project" load_entry_points("project") assert "Failed to load project commands" in caplog.text entry_points.return_value.select.assert_called_once_with( @@ -319,6 +320,7 @@ def test_global_groups(self, entry_points, entry_point): def test_global_error_is_caught(self, entry_points, entry_point, caplog): entry_point.load.side_effect = Exception() + entry_point.module = "global" load_entry_points("global") assert "Failed to load global commands" in caplog.text entry_points.return_value.select.assert_called_once_with( diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index 43426f0353..fadc49756c 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -12,7 +12,11 @@ from cookiecutter.exceptions import RepositoryCloneFailed from kedro import __version__ as version -from kedro.framework.cli.starters import _STARTER_ALIASES, TEMPLATE_PATH +from kedro.framework.cli.starters import ( + _OFFICIAL_STARTER_SPECS, + TEMPLATE_PATH, + KedroStarterSpec, +) FILES_IN_TEMPLATE = 32 @@ -78,10 +82,46 @@ def test_starter_list(fake_kedro_cli): result = CliRunner().invoke(fake_kedro_cli, ["starter", "list"]) assert result.exit_code == 0, result.output - for alias in _STARTER_ALIASES: + for alias in _OFFICIAL_STARTER_SPECS: assert alias in result.output +def test_starter_list_with_starter_plugin(fake_kedro_cli, entry_point): + """Check that `kedro starter list` prints out the plugin starters.""" + entry_point.load.return_value = [KedroStarterSpec("valid_starter", "valid_path")] + entry_point.module = "valid_starter_module" + result = CliRunner().invoke(fake_kedro_cli, ["starter", "list"]) + assert result.exit_code == 0, result.output + assert "valid_starter_module" in result.output + + +@pytest.mark.parametrize( + "specs,expected", + [ + ( + [{"alias": "valid_starter", "template_path": "valid_path"}], + "should be a 'KedroStarterSpec'", + ), + ( + [ + KedroStarterSpec("duplicate", "duplicate"), + KedroStarterSpec("duplicate", "duplicate"), + ], + "has been ignored as it is already defined by", + ), + ], +) +def test_starter_list_with_invalid_starter_plugin( + fake_kedro_cli, entry_point, specs, expected +): + """Check that `kedro starter list` prints out the plugin starters.""" + entry_point.load.return_value = specs + entry_point.module = "invalid_starter" + result = CliRunner().invoke(fake_kedro_cli, ["starter", "list"]) + assert result.exit_code == 0, result.output + assert expected in result.output + + def test_cookiecutter_json_matches_prompts_yml(): """Validate the contents of the default config file.""" cookiecutter_json = json.loads(