Migrate default project template to static project metadata

Uses `requirements.txt` for dev requirements in project template.

Fix gh-2280.
Fix gh-2519.

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>

features/environment.py

@@ -116,13 +116,18 @@ def _setup_minimal_env(context):
 
 def _install_project_requirements(context):
     install_reqs = (
-        Path(
-            "kedro/templates/project/{{ cookiecutter.repo_name }}/src/requirements.txt"
-        )
+        Path("kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt")
         .read_text(encoding="utf-8")
         .splitlines()
     )
-    install_reqs = [req for req in install_reqs if "{" not in req]
+    install_reqs = [
+        req
+        for req in install_reqs
+        if (req.strip())
+        and ("{" not in req)
+        and (not req.startswith("-e"))
+        and (not req.startswith("#"))
+    ]
     install_reqs.append(".[pandas.CSVDataSet]")
     call([context.pip, "install", *install_reqs], env=context.env)
     return context

features/steps/cli_steps.py

@@ -162,7 +162,7 @@ def create_config_file(context):
 @given("I have installed the project dependencies")
 def pip_install_dependencies(context):
     """Install project dependencies using pip."""
-    reqs_path = "src/requirements.txt"
+    reqs_path = "requirements.txt"
     res = run(
         [context.pip, "install", "-r", reqs_path],
         env=context.env,
@@ -410,7 +410,7 @@ def update_kedro_req(context: behave.runner.Context):
     """Replace kedro as a standalone requirement with a line
     that includes all of kedro's dependencies (-r kedro/requirements.txt)
     """
-    reqs_path = context.root_project_dir / "src" / "requirements.txt"
+    reqs_path = context.root_project_dir / "requirements.txt"
     kedro_reqs = f"-r {context.requirements_path.as_posix()}"
 
     if reqs_path.is_file():
@@ -428,7 +428,7 @@ def update_kedro_req(context: behave.runner.Context):
 
 @when("I add {dependency} to the requirements")
 def add_req(context: behave.runner.Context, dependency: str):
-    reqs_path = context.root_project_dir / "src" / "requirements.txt"
+    reqs_path = context.root_project_dir / "requirements.txt"
     if reqs_path.is_file():
         reqs_path.write_text(reqs_path.read_text() + "\n" + str(dependency) + "\n")
 
@@ -610,14 +610,14 @@ def check_docs_generated(context: behave.runner.Context):
 @then("requirements should be generated")
 def check_reqs_generated(context: behave.runner.Context):
     """Check that new project requirements are generated."""
-    reqs_path = context.root_project_dir / "src" / "requirements.lock"
+    reqs_path = context.root_project_dir / "requirements.lock"
     assert reqs_path.is_file()
     assert "This file is autogenerated by pip-compile" in reqs_path.read_text()
 
 
 @then("{dependency} should be in the requirements")
 def check_dependency_in_reqs(context: behave.runner.Context, dependency: str):
-    reqs_path = context.root_project_dir / "src" / "requirements.txt"
+    reqs_path = context.root_project_dir / "requirements.txt"
     assert dependency in reqs_path.read_text()
 
 

kedro/framework/cli/micropkg.py

@@ -165,7 +165,7 @@ def _pull_package(
         package_reqs = re.findall(reqs_element_pattern, list_reqs[0])
 
         if package_reqs:
-            requirements_txt = metadata.source_dir / "requirements.txt"
+            requirements_txt = metadata.project_path / "requirements.txt"
             _append_package_reqs(requirements_txt, package_reqs, package_name)
 
         _clean_pycache(temp_dir_path)
@@ -741,7 +741,7 @@ def _generate_sdist_file(
         # Build a setup.py on the fly
         try:
             install_requires = _make_install_requires(
-                package_source / "requirements.txt"  # type: ignore
+                metadata.project_path / "requirements.txt"  # type: ignore
             )
         except Exception as exc:
             click.secho("FAILED", fg="red")

kedro/framework/cli/project.py

@@ -88,9 +88,9 @@ def test(metadata: ProjectMetadata, args, **kwargs):  # pylint: disable=unused-a
     try:
         _check_module_importable("pytest")
     except KedroCliError as exc:
-        source_path = metadata.source_dir
+        project_path = metadata.project_path
         raise KedroCliError(
-            NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path))
+            NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(project_path))
         ) from exc
     python_call("pytest", args)
 
@@ -110,20 +110,22 @@ def lint(
     click.secho(deprecation_message, fg="red")
 
     source_path = metadata.source_dir
+    project_path = metadata.project_path
     package_name = metadata.package_name
-    files = files or (str(source_path / "tests"), str(source_path / package_name))
+    files = files or (str(project_path / "tests"), str(source_path / package_name))
 
     if "PYTHONPATH" not in os.environ:
         # isort needs the source path to be in the 'PYTHONPATH' environment
         # variable to treat it as a first-party import location
+        # NOTE: Actually, `pip install [-e] .` achieves the same
         os.environ["PYTHONPATH"] = str(source_path)  # pragma: no cover
 
     for module_name in ("flake8", "isort", "black"):
         try:
             _check_module_importable(module_name)
         except KedroCliError as exc:
             raise KedroCliError(
-                NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path))
+                NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(project_path))
             ) from exc
 
     python_call("black", ("--check",) + files if check_only else files)
@@ -199,10 +201,11 @@ def build_docs(metadata: ProjectMetadata, open_docs):
     click.secho(deprecation_message, fg="red")
 
     source_path = metadata.source_dir
+    project_path = metadata.project_path
     package_name = metadata.package_name
 
-    python_call("pip", ["install", str(source_path / "[docs]")])
-    python_call("pip", ["install", "-r", str(source_path / "requirements.txt")])
+    python_call("pip", ["install", str(project_path / "[docs]")])
+    python_call("pip", ["install", "-r", str(project_path / "requirements.txt")])
     python_call("ipykernel", ["install", "--user", f"--name={package_name}"])
     shutil.rmtree("docs/build", ignore_errors=True)
     call(
@@ -291,7 +294,7 @@ def activate_nbstripout(
     )
     click.secho(deprecation_message, fg="red")
 
-    source_path = metadata.source_dir
+    project_path = metadata.source_dir
     click.secho(
         (
             "Notebook output cells will be automatically cleared before committing"
@@ -304,7 +307,7 @@ def activate_nbstripout(
         _check_module_importable("nbstripout")
     except KedroCliError as exc:
         raise KedroCliError(
-            NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(source_path))
+            NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(project_path))
         ) from exc
 
     try:

kedro/templates/project/{{ cookiecutter.repo_name }}/README.md

@@ -64,25 +64,13 @@ After this, if you'd like to update your project requirements, please update `re
 > Jupyter, JupyterLab, and IPython are already included in the project requirements by default, so once you have run `pip install -r requirements.txt` you will not need to take any extra steps before you use them.
 
 ### Jupyter
-To use Jupyter notebooks in your Kedro project, you need to install Jupyter:
-
-```
-pip install jupyter
-```
-
 After installing Jupyter, you can start a local notebook server:
 
 ```
 kedro jupyter notebook
 ```
 
 ### JupyterLab
-To use JupyterLab, you need to install it:
-
-```
-pip install jupyterlab
-```
-
 You can also start JupyterLab:
 
 ```

kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -1,3 +1,38 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "{{ cookiecutter.python_package }}"
+dependencies = [
+    "kedro~={{ cookiecutter.kedro_version }}",
+]
+dynamic = ["version"]
+
+[project.scripts]
+{{ cookiecutter.repo_name }} = "{{ cookiecutter.python_package }}.__main__:main"
+
+[project.optional-dependencies]
+docs = [
+    "docutils<0.18.0",
+    "sphinx~=3.4.3",
+    "sphinx_rtd_theme==0.5.1",
+    "nbsphinx==0.8.1",
+    "nbstripout~=0.4",
+    "sphinx-autodoc-typehints==1.11.1",
+    "sphinx_copybutton==0.3.1",
+    "ipykernel>=5.3, <7.0",
+    "Jinja2<3.1.0",
+    "myst-parser~=0.17.2",
+]
+
+[tool.setuptools.dynamic]
+version = {attr = "{{ cookiecutter.python_package }}.__version__"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+namespaces = false
+
 [tool.kedro]
 package_name = "{{ cookiecutter.python_package }}"
 project_name = "{{ cookiecutter.project_name }}"

kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt

@@ -1,3 +1,7 @@
+# Install library code
+-e file:.
+
+# Development dependencies
 black~=22.0
 flake8>=3.7.9, <5.0
 ipython>=7.31.1, <8.0; python_version < '3.8'
@@ -6,8 +10,6 @@ isort~=5.0
 jupyter~=1.0
 jupyterlab_server>=2.11.1, <2.16.0
 jupyterlab~=3.0, <3.6.0
-kedro~={{ cookiecutter.kedro_version }}
-kedro-telemetry~=0.2.0
 nbstripout~=0.4
 pytest-cov~=3.0
 pytest-mock>=1.7.1, <2.0

tests/framework/cli/micropkg/conftest.py

@@ -26,7 +26,7 @@ def cleanup_micropackages(fake_repo_path, fake_package_path):
             if each.is_file():
                 each.unlink()
 
-        tests = fake_repo_path / "src" / "tests" / micropackage
+        tests = fake_repo_path / "tests" / micropackage
         if tests.is_dir():
             shutil.rmtree(str(tests))
 
@@ -35,7 +35,7 @@ def cleanup_micropackages(fake_repo_path, fake_package_path):
 def cleanup_pipelines(fake_repo_path, fake_package_path):
     pipes_path = fake_package_path / "pipelines"
     old_pipelines = {p.name for p in pipes_path.iterdir() if p.is_dir()}
-    requirements_txt = fake_repo_path / "src" / "requirements.txt"
+    requirements_txt = fake_repo_path / "requirements.txt"
     requirements = requirements_txt.read_text()
     yield
 
@@ -53,7 +53,7 @@ def cleanup_pipelines(fake_repo_path, fake_package_path):
             if each.is_file():
                 each.unlink()
 
-        tests = fake_repo_path / "src" / "tests" / "pipelines" / pipeline
+        tests = fake_repo_path / "tests" / "pipelines" / pipeline
         if tests.is_dir():
             shutil.rmtree(str(tests))
 

tests/framework/cli/test_project.py

@@ -121,7 +121,7 @@ def test_pytest_not_installed(
             fake_project_cli, ["test", "--random-arg", "value"], obj=fake_metadata
         )
         expected_message = NO_DEPENDENCY_MESSAGE.format(
-            module="pytest", src=str(fake_repo_path / "src")
+            module="pytest", src=str(fake_repo_path)
         )
 
         assert result.exit_code
@@ -148,7 +148,7 @@ def test_lint(
         assert not result.exit_code, result.stdout
 
         expected_files = files or (
-            str(fake_repo_path / "src/tests"),
+            str(fake_repo_path / "tests"),
             str(fake_repo_path / "src/dummy_package"),
         )
         expected_calls = [
@@ -185,7 +185,7 @@ def test_lint_check_only(
         assert not result.exit_code, result.stdout
 
         expected_files = files or (
-            str(fake_repo_path / "src/tests"),
+            str(fake_repo_path / "tests"),
             str(fake_repo_path / "src/dummy_package"),
         )
         expected_calls = [
@@ -217,7 +217,7 @@ def test_import_not_installed(
 
         result = CliRunner().invoke(fake_project_cli, ["lint"], obj=fake_metadata)
         expected_message = NO_DEPENDENCY_MESSAGE.format(
-            module=module_name, src=str(fake_repo_path / "src")
+            module=module_name, src=str(fake_repo_path)
         )
 
         assert result.exit_code, result.stdout
@@ -354,7 +354,7 @@ def test_happy_path(
                 mocker.call("pip", ["install", str(fake_repo_path / "src/[docs]")]),
                 mocker.call(
                     "pip",
-                    ["install", "-r", str(fake_repo_path / "src/requirements.txt")],
+                    ["install", "-r", str(fake_repo_path / "requirements.txt")],
                 ),
                 mocker.call("ipykernel", ["install", "--user", "--name=dummy_package"]),
             ]

tests/framework/cli/test_starters.py

@@ -17,7 +17,7 @@
     KedroStarterSpec,
 )
 
-FILES_IN_TEMPLATE = 31
+FILES_IN_TEMPLATE = 30
 
 
 @pytest.fixture
@@ -70,9 +70,7 @@ def _assert_template_ok(
     assert (full_path / ".gitignore").is_file()
     assert project_name in (full_path / "README.md").read_text(encoding="utf-8")
     assert "KEDRO" in (full_path / ".gitignore").read_text(encoding="utf-8")
-    assert kedro_version in (full_path / "src" / "requirements.txt").read_text(
-        encoding="utf-8"
-    )
+    assert kedro_version in (full_path / "pyproject.toml").read_text(encoding="utf-8")
     assert (full_path / "src" / python_package / "__init__.py").is_file()