Do not modify MANIFEST.in on install

.actions/setup_tools.py

@@ -157,6 +157,7 @@ def distribute_version(src_folder: str, ver_file: str = "version.info") -> None:
     ver_template = os.path.join(src_folder, ver_file)
     for fpath in ls_ver:
         fpath = os.path.join(os.path.dirname(fpath), ver_file)
+        print("Distributing the version to", fpath)
         if os.path.isfile(fpath):
             os.remove(fpath)
         shutil.copy2(ver_template, fpath)

.github/actions/pkg-check/action.yml

@@ -33,9 +33,9 @@ runs:
 
   - name: Check package
     run: |
+      git diff --exit-code || exit $?  # make sure there are no local unstaged changes
       ls -l dist/
       twine check dist/*
-      # python setup.py clean
     shell: bash
 
   - name: Unzip packages

.github/workflows/docs-checks.yml

@@ -79,8 +79,7 @@ jobs:
         # This is needed as App docs is heavily using/referring to lightning package
         if: ${{ matrix.pkg-name == 'app' }}
         run: |
-          pip install -e . -U -v  -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
-          git checkout -- .
+          pip install -e . -U -v -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
 
       - name: Adjust docs refs
         if: ${{ matrix.pkg-name == 'app' }}

setup.py

@@ -39,11 +39,15 @@
     b) with a parameterization build desired packages in to standard `dist/` folder
     c) validate packages and publish to PyPI
 """
+import contextlib
 import os
+import tempfile
 from importlib.util import module_from_spec, spec_from_file_location
 from types import ModuleType
+from typing import Generator, Optional
 
-from setuptools import setup
+import setuptools
+import setuptools.command.egg_info
 
 _PACKAGE_NAME = os.environ.get("PACKAGE_NAME")
 _PACKAGE_MAPPING = {
@@ -69,36 +73,82 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     return py
 
 
+def _named_temporary_file(directory: Optional[str] = None) -> str:
+    # `tempfile.NamedTemporaryFile` has issues in Windows
+    # https://github.com/deepchem/deepchem/issues/707#issuecomment-556002823
+    if directory is None:
+        directory = tempfile.gettempdir()
+    return os.path.join(directory, os.urandom(24).hex())
+
+
+@contextlib.contextmanager
+def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator:
+    if aggregate:
+        # aggregate all MANIFEST.in contents into a single temporary file
+        manifest_path = _named_temporary_file(manifest_dir)
+        mapping = _PACKAGE_MAPPING.copy()
+        del mapping["lightning"]
+        lines = ["include src/lightning/version.info\n"]
+        for pkg in mapping.values():
+            with open(os.path.join(_PATH_SRC, pkg, "MANIFEST.in")) as fh:
+                lines.extend(fh.readlines())
+        # convert lightning_foo to lightning/foo
+        for new, old in mapping.items():
+            lines = [line.replace(old, f"lightning/{new}") for line in lines]
+        with open(manifest_path, mode="w") as fp:
+            fp.writelines(lines)
+    else:
+        manifest_path = os.path.join(manifest_dir, "MANIFEST.in")
+        assert os.path.exists(manifest_path)
+    # avoid error: setup script specifies an absolute path
+    manifest_path = os.path.relpath(manifest_path, _PATH_ROOT)
+    print("Set manifest path to", manifest_path)
+    setuptools.command.egg_info.manifest_maker.template = manifest_path
+    yield
+    # cleanup
+    setuptools.command.egg_info.manifest_maker.template = "MANIFEST.in"
+    if aggregate:
+        os.remove(manifest_path)
+
+
 if __name__ == "__main__":
     setup_tools = _load_py_module(name="setup_tools", location=os.path.join(_PATH_ROOT, ".actions", "setup_tools.py"))
     assistant = _load_py_module(name="assistant", location=os.path.join(_PATH_ROOT, ".actions", "assistant.py"))
 
-    if os.path.exists(_PATH_SRC):  # not a wheel install
+    if os.path.exists(_PATH_SRC):
         # copy the version information to all packages
         setup_tools.distribute_version(_PATH_SRC)
 
     package_to_install = _PACKAGE_NAME or "lightning"
     print(f"Installing the {package_to_install} package")  # requires `-v` to appear
+    is_wheel_install = "PEP517_BUILD_BACKEND" in os.environ
+    print("is_wheel_install:", is_wheel_install)
+    if package_to_install not in _PACKAGE_MAPPING or (not is_wheel_install and _PACKAGE_NAME is None):
+        raise ValueError(f"Unexpected package name: {_PACKAGE_NAME}. Possible choices are: {list(_PACKAGE_MAPPING)}")
+    is_wheel_install &= _PACKAGE_NAME is None
+
     if package_to_install == "lightning":  # install everything
         # merge all requirements files
         setup_tools._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS)
         # replace imports and copy the code
         assistant.create_mirror_package(_PATH_SRC, _PACKAGE_MAPPING)
-    elif package_to_install not in _PACKAGE_MAPPING:
-        raise ValueError(f"Unexpected package name: {_PACKAGE_NAME}. Possible choices are: {list(_PACKAGE_MAPPING)}")
 
-    # if `_PACKAGE_NAME` is not set, iterate over all possible packages until we find one that can be installed.
-    # this is useful for installing existing wheels, as the user wouldn't set this environment variable, but the wheel
-    # should have included only the relevant files of the package to install
-    possible_packages = _PACKAGE_MAPPING.values() if _PACKAGE_NAME is None else [_PACKAGE_MAPPING[_PACKAGE_NAME]]
+    # if it's a wheel install (hence _PACKAGE_NAME should not be set), iterate over all possible packages until we find
+    # one that can be installed. the wheel should have included only the relevant files of the package to install
+    possible_packages = _PACKAGE_MAPPING.values() if is_wheel_install else [_PACKAGE_MAPPING[_PACKAGE_NAME]]
     for pkg in possible_packages:
-        pkg_setup = os.path.join(_PATH_SRC, pkg, "__setup__.py")
+        pkg_path = os.path.join(_PATH_SRC, pkg)
+        pkg_setup = os.path.join(pkg_path, "__setup__.py")
         if os.path.exists(pkg_setup):
             print(f"{pkg_setup} exists. Running `setuptools.setup`")
             setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup)
-            setup_module._adjust_manifest(pkg_name=pkg)
-            setup_args = setup_module._setup_args(pkg_name=pkg)
-            setup(**setup_args)
+            setup_args = setup_module._setup_args()
+            if is_wheel_install:
+                setuptools.setup(**setup_args)
+            else:
+                # we are installing from source, set the correct manifest path
+                with _set_manifest_path(pkg_path, aggregate=pkg == "lightning"):
+                    setuptools.setup(**setup_args)
             break
     else:
         raise RuntimeError(f"Something's wrong, no package was installed. Package name: {_PACKAGE_NAME}")

src/lightning/__setup__.py

@@ -50,29 +50,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**kwargs: Any) -> None:
-    # todo: consider rather aggregation of particular manifest adjustments
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = [ln.rstrip() for ln in fp.readlines()]
-    lines += [
-        "recursive-include src/lightning *.md",
-        "recursive-include requirements *.txt",
-        "recursive-include src/lightning/app/ui *",
-        "recursive-include src/lightning/cli/*-template *",  # Add templates as build-in
-        "include src/lightning/version.info" + os.linesep,
-        "include src/lightning/app/components/serve/catimage.png" + os.linesep,
-        # fixme: this is strange, this shall work with setup find package - include
-        "prune src/lightning_app",
-        "prune src/lightning_lite",
-        "prune src/pytorch_lightning",
-    ]
-    with open(manifest_path, "w") as fp:
-        fp.writelines([ln + os.linesep for ln in lines])
-
-
-def _setup_args(**kwargs: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))
     _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py"))
     _long_description = _SETUP_TOOLS.load_readme_description(

src/lightning_app/MANIFEST.in

@@ -0,0 +1,11 @@
+include src/version.info
+include src/lightning_app/version.info
+include src/lightning_app/CHANGELOG.md
+include src/lightning_app/README.md
+recursive-include requirements/app *.txt
+include .actions/setup_tools.py
+include .actions/assistant.py
+recursive-include src/lightning_app/cli/*-template *
+# TODO: remove this once lightning-ui package is ready as a dependency
+recursive-include src/lightning_app/ui *
+include src/lightning/app/components/serve/catimage.png

src/lightning_app/__setup__.py

@@ -41,29 +41,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**__: Any) -> None:
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = fp.readlines()
-    lines += [
-        "recursive-exclude src *.md" + os.linesep,
-        "recursive-exclude requirements *.txt" + os.linesep,
-        "recursive-include src/lightning_app *.md" + os.linesep,
-        "include src/lightning_app/components/serve/catimage.png" + os.linesep,
-        "recursive-include requirements/app *.txt" + os.linesep,
-        "recursive-include src/lightning_app/cli/*-template *" + os.linesep,  # Add templates
-        "include src/lightning_app/version.info" + os.linesep,
-    ]
-
-    # TODO: remove this once lightning-ui package is ready as a dependency
-    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
-
-    with open(manifest_path, "w") as fp:
-        fp.writelines(lines)
-
-
-def _setup_args(**__: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))

src/lightning_app/testing/testing.py

@@ -420,8 +420,6 @@ def run_app_in_cloud(
             if resp.status_code == 200:
                 break
 
-        print(f"The Lightning Id Name : [bold magenta]{app_id}[/bold magenta]")
-
         logs_api_client = _LightningLogsSocketAPI(client.api_client)
 
         def fetch_logs(component_names: Optional[List[str]] = None) -> Generator:

src/lightning_app/utilities/packaging/lightning_utils.py

@@ -102,9 +102,7 @@ def _prepare_lightning_wheels_and_requirements(root: Path, package_name: str = "
     if not get_dist_path_if_editable_install(package_name):
         return
 
-    # this is patch for installing `lightning-app` as standalone package
-    if package_name == "lightning_app":
-        os.environ["PACKAGE_NAME"] = "app"
+    os.environ["PACKAGE_NAME"] = "app" if package_name == "lightning" + "_app" else "lightning"
 
     # Packaging the Lightning codebase happens only inside the `lightning` repo.
     git_dir_name = get_dir_name() if check_github_repository() else None

src/lightning_lite/MANIFEST.in

@@ -0,0 +1,7 @@
+include src/version.info
+include src/lightning_lite/version.info
+include src/lightning_lite/CHANGELOG.md
+include src/lightning_lite/README.md
+recursive-include requirements/lite *.txt
+include .actions/setup_tools.py
+include .actions/assistant.py

src/lightning_lite/__setup__.py

@@ -42,27 +42,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**__: Any) -> None:
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = fp.readlines()
-    lines += [
-        "recursive-exclude src *.md" + os.linesep,
-        "recursive-exclude requirements *.txt" + os.linesep,
-        "recursive-include requirements/lite *.txt" + os.linesep,
-        "recursive-include src/lightning_lite *.md" + os.linesep,
-        "include src/lightning_lite/version.info" + os.linesep,
-    ]
-
-    # TODO: remove this once lightning-ui package is ready as a dependency
-    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
-
-    with open(manifest_path, "w") as fp:
-        fp.writelines(lines)
-
-
-def _setup_args(**__: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))

src/pytorch_lightning/MANIFEST.in

@@ -0,0 +1,13 @@
+include src/version.info
+# distribute the lite source code inside PL
+include src/lightning_lite/version.info
+include src/lightning_lite/CHANGELOG.md
+recursive-include requirements/lite *.txt
+include src/pytorch_lightning/version.info
+include src/pytorch_lightning/CHANGELOG.md
+include src/pytorch_lightning/README.md
+recursive-include requirements/pytorch *.txt
+include .actions/setup_tools.py
+include .actions/assistant.py
+include *.cff  # citation info
+include src/pytorch_lightning/py.typed  # marker file for PEP 561

src/pytorch_lightning/__setup__.py

@@ -44,27 +44,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**__: Any) -> None:
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = fp.readlines()
-    lines += [
-        "recursive-exclude src *.md" + os.linesep,
-        "recursive-exclude requirements *.txt" + os.linesep,
-        "recursive-include requirements/lite *.txt" + os.linesep,
-        "recursive-include src/lightning_lite *.md" + os.linesep,
-        "include src/lightning_lite/version.info" + os.linesep,
-        "recursive-include src/pytorch_lightning *.md" + os.linesep,
-        "recursive-include requirements/pytorch *.txt" + os.linesep,
-        "include src/pytorch_lightning/version.info" + os.linesep,
-        "include src/pytorch_lightning/py.typed" + os.linesep,  # marker file for PEP 561
-    ]
-    with open(manifest_path, "w") as fp:
-        fp.writelines(lines)
-
-
-def _setup_args(**__: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))

tests/tests_app/utilities/packaging/test_lightning_utils.py

@@ -1,3 +1,4 @@
+import glob
 import os
 from unittest import mock
 
@@ -9,27 +10,21 @@
 from lightning_app.utilities.packaging.lightning_utils import (
     _prepare_lightning_wheels_and_requirements,
     _verify_lightning_version,
+    get_dist_path_if_editable_install,
 )
 
 
-# TODO: Resolve this sensitive test.
-@pytest.mark.skipif(True, reason="Currently broken")
+@pytest.mark.skipif(not module_available("lightning"), reason="TODO: should work for lightning_app too")
 def test_prepare_lightning_wheels_and_requirement(tmpdir):
     """This test ensures the lightning source gets packaged inside the lightning repo."""
-
-    package_name = "lightning" if module_available("lightning") else "lightning-app"
-
-    if package_name == "lightning":
-        from lightning.__version__ import version
-
-        tar_name = f"lightning-{version}.tar.gz"
-    else:
-        from lightning_app.__version__ import version
-
-        tar_name = f"lightning-app-{version}.tar.gz"
+    package_name = "lightning"
+    if not get_dist_path_if_editable_install(package_name):
+        pytest.skip("Requires --editable install")
 
     cleanup_handle = _prepare_lightning_wheels_and_requirements(tmpdir, package_name=package_name)
-    assert sorted(os.listdir(tmpdir))[0] == tar_name
+    assert len(os.listdir(tmpdir)) == 1
+    assert len(glob.glob(str(tmpdir / "lightning-*.tar.gz"))) == 1
+
     cleanup_handle()
     assert os.listdir(tmpdir) == []