From d1ee9414559161e440e4d40fdccc253ce057c38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 10 Jul 2023 17:16:48 +0200 Subject: [PATCH 1/8] Remove leftover reference to micropkg setup.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- tests/framework/cli/micropkg/test_micropkg_requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/framework/cli/micropkg/test_micropkg_requirements.py b/tests/framework/cli/micropkg/test_micropkg_requirements.py index b0070a1bee..4b424073b7 100644 --- a/tests/framework/cli/micropkg/test_micropkg_requirements.py +++ b/tests/framework/cli/micropkg/test_micropkg_requirements.py @@ -249,8 +249,8 @@ def test_empty_pipeline_requirements_txt( def test_complex_requirements( self, requirement, fake_project_cli, fake_metadata, fake_package_path ): - """Options that are valid in requirements.txt but cannot be packaged using - setup.py.""" + """Options that are valid in requirements.txt but cannot be packaged in + pyproject.toml.""" self.call_pipeline_create(fake_project_cli, fake_metadata) pipeline_requirements_txt = ( fake_package_path / "pipelines" / PIPELINE_NAME / "requirements.txt" From 9b3e63622f8cd51611ca0c4156ab1b1f0fe88443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 10 Jul 2023 16:52:48 +0200 Subject: [PATCH 2/8] Replace outdated extras requirement with kedro-datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- features/environment.py | 2 +- .../{{ cookiecutter.repo_name }}/src/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/features/environment.py b/features/environment.py index 172dfd006a..dcf47fc118 100644 --- a/features/environment.py +++ b/features/environment.py @@ -124,6 +124,6 @@ def _install_project_requirements(context): .splitlines() ) install_reqs = [req for req in install_reqs if "{" not in req] - install_reqs.append(".[pandas.CSVDataSet]") + install_reqs.append("kedro-datasets[pandas.CSVDataSet]") call([context.pip, "install", *install_reqs], env=context.env) return context diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt index 7e6f29ac16..e18878f6db 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt @@ -6,7 +6,7 @@ isort~=5.0 jupyter~=1.0 jupyterlab_server>=2.11.1, <2.16.0 jupyterlab~=3.0, <3.6.0 -kedro[pandas.CSVDataSet]=={{ cookiecutter.kedro_version }} +kedro-datasets[pandas.CSVDataSet]~=1.2 kedro-telemetry~=0.2.0 nbstripout~=0.4 pytest-cov~=3.0 From 83010f6a6d835e505becd9d50039ad146752556a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 10 Jul 2023 16:53:01 +0200 Subject: [PATCH 3/8] Remove outdated mention to environment.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- .../steps/test_starter/{{ cookiecutter.repo_name }}/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/README.md b/features/steps/test_starter/{{ cookiecutter.repo_name }}/README.md index 8041d41dd9..fd206a315d 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/README.md +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/README.md @@ -17,7 +17,7 @@ In order to get the best out of the template: ## How to install dependencies -Declare any dependencies in `src/requirements.txt` for `pip` installation and `src/environment.yml` for `conda` installation. +Declare any dependencies in `src/requirements.txt` for `pip` installation. To install them, run: From edc32ce06d90fc0e90419e4de5b803051652be98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 10 Jul 2023 16:53:25 +0200 Subject: [PATCH 4/8] Move flake8 configuration to dedicated file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- docs/source/deployment/airflow_astronomer.md | 2 +- docs/source/development/linting.md | 2 +- docs/source/get_started/kedro_concepts.md | 2 +- docs/source/kedro_project_setup/starters.md | 2 +- .../steps/test_starter/{{ cookiecutter.repo_name }}/.flake8 | 3 +++ .../test_starter/{{ cookiecutter.repo_name }}/setup.cfg | 5 ----- .../{{ cookiecutter.repo_name }}/{setup.cfg => .flake8} | 0 7 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 features/steps/test_starter/{{ cookiecutter.repo_name }}/.flake8 delete mode 100644 features/steps/test_starter/{{ cookiecutter.repo_name }}/setup.cfg rename kedro/templates/project/{{ cookiecutter.repo_name }}/{setup.cfg => .flake8} (100%) diff --git a/docs/source/deployment/airflow_astronomer.md b/docs/source/deployment/airflow_astronomer.md index 307e9ab903..b21408156b 100644 --- a/docs/source/deployment/airflow_astronomer.md +++ b/docs/source/deployment/airflow_astronomer.md @@ -74,7 +74,7 @@ To follow this tutorial, ensure you have the following: ├── plugins ├── pyproject.toml ├── requirements.txt - ├── setup.cfg + ├── .flake8 └── src ``` diff --git a/docs/source/development/linting.md b/docs/source/development/linting.md index d795086b51..ca53d7f49d 100644 --- a/docs/source/development/linting.md +++ b/docs/source/development/linting.md @@ -60,7 +60,7 @@ It is a good practice to [split your line when it is too long](https://beta.ruff #### Configure `flake8` -Store your `flake8` configuration in a file named `setup.cfg` within your project root. The Kedro starters use the [following configuration](https://github.com/kedro-org/kedro-starters/blob/main/pandas-iris/%7B%7B%20cookiecutter.repo_name%20%7D%7D/setup.cfg): +Store your `flake8` configuration in a file named `.flake8` within your project root. The Kedro starters use the [following configuration](https://github.com/kedro-org/kedro-starters/blob/main/pandas-iris/%7B%7B%20cookiecutter.repo_name%20%7D%7D/.flake8): ```text [flake8] diff --git a/docs/source/get_started/kedro_concepts.md b/docs/source/get_started/kedro_concepts.md index 4a6d771da0..67f9bf84a9 100644 --- a/docs/source/get_started/kedro_concepts.md +++ b/docs/source/get_started/kedro_concepts.md @@ -72,7 +72,7 @@ project-dir # Parent directory of the template ├── notebooks # Project-related Jupyter notebooks (can be used for experimental code before moving the code to src) ├── pyproject.toml # Identifies the project root and contains configuration information ├── README.md # Project README -├── setup.cfg # Configuration options for `pytest` when doing `kedro test` and for the `isort` utility when doing `kedro lint` +├── .flake8 # Configuration options for `flake8` (linting) └── src # Project source code ``` diff --git a/docs/source/kedro_project_setup/starters.md b/docs/source/kedro_project_setup/starters.md index 4ad60cb0c2..7505151935 100644 --- a/docs/source/kedro_project_setup/starters.md +++ b/docs/source/kedro_project_setup/starters.md @@ -155,7 +155,7 @@ Here is the layout of the project as a Cookiecutter template: ├── docs # Project documentation ├── notebooks # Project related Jupyter notebooks (can be used for experimental code before moving the code to src) ├── README.md # Project README -├── setup.cfg # Configuration options for tools e.g. `pytest` or `black` +├── .flake8 # Configuration options for `flake8` (linting) └── src # Project source code └── {{ cookiecutter.python_package }} ├── __init.py__ diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/.flake8 b/features/steps/test_starter/{{ cookiecutter.repo_name }}/.flake8 new file mode 100644 index 0000000000..8dd399ab55 --- /dev/null +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203 diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/setup.cfg b/features/steps/test_starter/{{ cookiecutter.repo_name }}/setup.cfg deleted file mode 100644 index 3edf923e16..0000000000 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -; Flake8 will be moved in `pyproject.toml` once the library -; starts supporting such config source -[flake8] -max-line-length = 88 -extend-ignore = E203 diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/setup.cfg b/kedro/templates/project/{{ cookiecutter.repo_name }}/.flake8 similarity index 100% rename from kedro/templates/project/{{ cookiecutter.repo_name }}/setup.cfg rename to kedro/templates/project/{{ cookiecutter.repo_name }}/.flake8 From 4c87a7554d86e98a9a9a96e0faac9d9a154d9d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 10 Jul 2023 17:08:02 +0200 Subject: [PATCH 5/8] Replace setup.py with pyproject.toml in default project template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- docs/source/kedro_project_setup/starters.md | 2 +- .../src/pyproject.toml | 31 +++++++++++++++ .../{{ cookiecutter.repo_name }}/src/setup.py | 39 ------------------- .../src/pyproject.toml | 31 +++++++++++++++ .../{{ cookiecutter.repo_name }}/src/setup.py | 39 ------------------- 5 files changed, 63 insertions(+), 79 deletions(-) create mode 100644 features/steps/test_starter/{{ cookiecutter.repo_name }}/src/pyproject.toml delete mode 100644 features/steps/test_starter/{{ cookiecutter.repo_name }}/src/setup.py create mode 100644 kedro/templates/project/{{ cookiecutter.repo_name }}/src/pyproject.toml delete mode 100644 kedro/templates/project/{{ cookiecutter.repo_name }}/src/setup.py diff --git a/docs/source/kedro_project_setup/starters.md b/docs/source/kedro_project_setup/starters.md index 7505151935..305fe1de00 100644 --- a/docs/source/kedro_project_setup/starters.md +++ b/docs/source/kedro_project_setup/starters.md @@ -164,7 +164,7 @@ Here is the layout of the project as a Cookiecutter template: ├── __main__.py └── settings.py ├── requirements.txt - ├── setup.py + ├── pyproject.toml └── tests ``` diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/pyproject.toml b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/pyproject.toml new file mode 100644 index 0000000000..ea581e7028 --- /dev/null +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "{{ cookiecutter.python_package }}" +version = "0.1" +dynamic = ["dependencies"] + +[tool.setuptools.packages.find] +exclude = ["tests"] + +[project.scripts] +{{ cookiecutter.repo_name }} = "{{ cookiecutter.python_package }}.__main__:main" + +[project.optional-dependencies] +docs = [ + "docutils<0.18.0", + "sphinx~=3.4.3", + "sphinx_rtd_theme==0.5.1", + "nbsphinx==0.8.1", + "nbstripout~=0.4", + "sphinx-autodoc-typehints==1.11.1", + "sphinx_copybutton==0.3.1", + "ipykernel>=5.3, <7.0", + "Jinja2<3.1.0", + "myst-parser~=0.17.2", +] + +[tool.setuptools.dynamic] +dependencies = {file = "requirements.txt"} diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/setup.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/setup.py deleted file mode 100644 index af5b101519..0000000000 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -from setuptools import find_packages, setup - -entry_point = ( - "{{ cookiecutter.repo_name }} = {{ cookiecutter.python_package }}.__main__:main" -) - - -# get the dependencies and installs -with open("requirements.txt", encoding="utf-8") as f: - # Make sure we strip all comments and options (e.g "--extra-index-url") - # that arise from a modified pip.conf file that configure global options - # when running kedro build-reqs - requires = [] - for line in f: - req = line.split("#", 1)[0].strip() - if req and not req.startswith("-r"): - requires.append(req) - -setup( - name="{{ cookiecutter.python_package }}", - version="0.1", - packages=find_packages(exclude=["tests"]), - entry_points={"console_scripts": [entry_point]}, - install_requires=requires, - extras_require={ - "docs": [ - "docutils<0.18.0", - "sphinx~=3.4.3", - "sphinx_rtd_theme==0.5.1", - "nbsphinx==0.8.1", - "nbstripout~=0.4", - "sphinx-autodoc-typehints==1.11.1", - "sphinx_copybutton==0.3.1", - "ipykernel>=5.3, <7.0", - "Jinja2<3.1.0", - "myst-parser~=0.17.2", - ] - }, -) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/pyproject.toml b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/pyproject.toml new file mode 100644 index 0000000000..ea581e7028 --- /dev/null +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "{{ cookiecutter.python_package }}" +version = "0.1" +dynamic = ["dependencies"] + +[tool.setuptools.packages.find] +exclude = ["tests"] + +[project.scripts] +{{ cookiecutter.repo_name }} = "{{ cookiecutter.python_package }}.__main__:main" + +[project.optional-dependencies] +docs = [ + "docutils<0.18.0", + "sphinx~=3.4.3", + "sphinx_rtd_theme==0.5.1", + "nbsphinx==0.8.1", + "nbstripout~=0.4", + "sphinx-autodoc-typehints==1.11.1", + "sphinx_copybutton==0.3.1", + "ipykernel>=5.3, <7.0", + "Jinja2<3.1.0", + "myst-parser~=0.17.2", +] + +[tool.setuptools.dynamic] +dependencies = {file = "requirements.txt"} diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/setup.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/setup.py deleted file mode 100644 index 8e62d661f8..0000000000 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -from setuptools import find_packages, setup - -entry_point = ( - "{{ cookiecutter.repo_name }} = {{ cookiecutter.python_package }}.__main__:main" -) - - -# get the dependencies and installs -with open("requirements.txt", encoding="utf-8") as f: - # Make sure we strip all comments and options (e.g "--extra-index-url") - # that arise from a modified pip.conf file that configure global options - # when running kedro build-reqs - requires = [] - for line in f: - req = line.split("#", 1)[0].strip() - if req and not req.startswith("--"): - requires.append(req) - -setup( - name="{{ cookiecutter.python_package }}", - version="0.1", - packages=find_packages(exclude=["tests"]), - entry_points={"console_scripts": [entry_point]}, - install_requires=requires, - extras_require={ - "docs": [ - "docutils<0.18.0", - "sphinx~=3.4.3", - "sphinx_rtd_theme==0.5.1", - "nbsphinx==0.8.1", - "nbstripout~=0.4", - "sphinx-autodoc-typehints==1.11.1", - "sphinx_copybutton==0.3.1", - "ipykernel>=5.3, <7.0", - "Jinja2<3.1.0", - "myst-parser~=0.17.2", - ] - }, -) From 6c1c4be805f53063cbdf146c7e444ea346de11be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 28 Jul 2023 09:44:37 +0200 Subject: [PATCH 6/8] Adapt link to moving file in docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- docs/source/conf.py | 4 +++- docs/source/development/linting.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 804bbbbfa9..5aec31bc55 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -227,7 +227,9 @@ "https://opensource.org/license/apache2-0-php/", "https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password", "https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/api/snowflake.snowpark.DataFrameWriter.saveAsTable.html", - "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors" + "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors", + # temporarily, until the file actually exists in `main` + "https://github.com/kedro-org/kedro/blob/main/kedro/templates/project/%7B%7B%20cookiecutter.repo_name%20%7D%7D/.flake8", ] # retry before render a link broken (fix for "too many requests") diff --git a/docs/source/development/linting.md b/docs/source/development/linting.md index ca53d7f49d..a8bdbc0c44 100644 --- a/docs/source/development/linting.md +++ b/docs/source/development/linting.md @@ -60,7 +60,7 @@ It is a good practice to [split your line when it is too long](https://beta.ruff #### Configure `flake8` -Store your `flake8` configuration in a file named `.flake8` within your project root. The Kedro starters use the [following configuration](https://github.com/kedro-org/kedro-starters/blob/main/pandas-iris/%7B%7B%20cookiecutter.repo_name%20%7D%7D/.flake8): +Store your `flake8` configuration in a file named `.flake8` within your project root. The Kedro default project template use the [following configuration](https://github.com/kedro-org/kedro/blob/main/kedro/templates/project/%7B%7B%20cookiecutter.repo_name%20%7D%7D/.flake8): ```text [flake8] From 97524f6eaf4766adb6bec5dfcf1b865fde7105a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 16 Aug 2023 18:21:52 +0200 Subject: [PATCH 7/8] Revert "Replace outdated extras requirement with kedro-datasets" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit dfa8a9e8377ca60dcc3026e93087c5538791a0b5. Signed-off-by: Juan Luis Cano Rodríguez --- features/environment.py | 2 +- .../{{ cookiecutter.repo_name }}/src/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/features/environment.py b/features/environment.py index dcf47fc118..172dfd006a 100644 --- a/features/environment.py +++ b/features/environment.py @@ -124,6 +124,6 @@ def _install_project_requirements(context): .splitlines() ) install_reqs = [req for req in install_reqs if "{" not in req] - install_reqs.append("kedro-datasets[pandas.CSVDataSet]") + install_reqs.append(".[pandas.CSVDataSet]") call([context.pip, "install", *install_reqs], env=context.env) return context diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt index e18878f6db..7e6f29ac16 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/requirements.txt @@ -6,7 +6,7 @@ isort~=5.0 jupyter~=1.0 jupyterlab_server>=2.11.1, <2.16.0 jupyterlab~=3.0, <3.6.0 -kedro-datasets[pandas.CSVDataSet]~=1.2 +kedro[pandas.CSVDataSet]=={{ cookiecutter.kedro_version }} kedro-telemetry~=0.2.0 nbstripout~=0.4 pytest-cov~=3.0 From e9c6137ed9b05f398e44ba8e085b85a40d0df339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 16 Aug 2023 18:23:54 +0200 Subject: [PATCH 8/8] Add release notes for pyproject.toml change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- RELEASE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 169485d03e..95e51359a9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,8 @@ ## Bug fixes and other changes * Updated `kedro pipeline create` and `kedro catalog create` to use new `/conf` file structure. +* Converted `setup.py` in default template to `pyproject.toml` and moved flake8 configuration + to dedicated file `.flake8`. ## Documentation changes * Revised the `data` section to restructure beginner and advanced pages about the Data Catalog and datasets.