Revert "Do not modify MANIFEST.in on install (#15549)" (#15644)

This reverts commit 4ea44dd.

Author: carmocca

.github/actions/pkg-check/action.yml

@@ -33,9 +33,9 @@ runs:
 
   - name: Check package
     run: |
-      git diff --exit-code || exit $?  # make sure there are no local unstaged changes
       ls -l dist/
       twine check dist/*
+      # python setup.py clean
     shell: bash
 
   - name: Unzip packages

.github/workflows/docs-checks.yml

@@ -75,6 +75,7 @@ jobs:
         if: ${{ matrix.pkg-name == 'app' }}
         run: |
           pip install -e . -U  -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
+          git checkout -- .
 
       - name: Adjust docs refs
         if: ${{ matrix.pkg-name == 'app' }}

MANIFEST.in

@@ -0,0 +1,6 @@
+exclude *.toml  # project config
+exclude requirements.txt
+exclude __pycache__
+include .actions/setup_tools.py
+include .actions/assistant.py
+include *.cff  # citation info

setup.py

@@ -39,15 +39,11 @@
     b) with a parameterization build desired packages in to standard `dist/` folder
     c) validate packages and publish to PyPI
 """
-import contextlib
 import os
-import tempfile
 from importlib.util import module_from_spec, spec_from_file_location
 from types import ModuleType
-from typing import Generator, Optional
 
-import setuptools
-import setuptools.command.egg_info
+from setuptools import setup
 
 _PACKAGE_NAME = os.environ.get("PACKAGE_NAME")
 _PACKAGE_MAPPING = {
@@ -73,43 +69,6 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     return py
 
 
-def _named_temporary_file(directory: Optional[str] = None) -> str:
-    # `tempfile.NamedTemporaryFile` has issues in Windows
-    # https://github.com/deepchem/deepchem/issues/707#issuecomment-556002823
-    if directory is None:
-        directory = tempfile.gettempdir()
-    return os.path.join(directory, os.urandom(24).hex())
-
-
-@contextlib.contextmanager
-def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator:
-    if aggregate:
-        # aggregate all MANIFEST.in contents into a single temporary file
-        manifest_path = _named_temporary_file(manifest_dir)
-        mapping = _PACKAGE_MAPPING.copy()
-        del mapping["lightning"]
-        lines = []
-        for pkg in mapping.values():
-            with open(os.path.join(_PATH_SRC, pkg, "MANIFEST.in")) as fh:
-                lines.extend(fh.readlines())
-        # convert lightning_foo to lightning/foo
-        for new, old in mapping.items():
-            lines = [line.replace(old, f"lightning/{new}") for line in lines]
-        with open(manifest_path, mode="w") as fp:
-            fp.writelines(lines)
-    else:
-        manifest_path = os.path.join(manifest_dir, "MANIFEST.in")
-        assert os.path.exists(manifest_path)
-    # avoid error: setup script specifies an absolute path
-    manifest_path = os.path.relpath(manifest_path, _PATH_ROOT)
-    setuptools.command.egg_info.manifest_maker.template = manifest_path
-    yield
-    # cleanup
-    setuptools.command.egg_info.manifest_maker.template = "MANIFEST.in"
-    if aggregate:
-        os.remove(manifest_path)
-
-
 if __name__ == "__main__":
     setup_tools = _load_py_module(name="setup_tools", location=os.path.join(_PATH_ROOT, ".actions", "setup_tools.py"))
     assistant = _load_py_module(name="assistant", location=os.path.join(_PATH_ROOT, ".actions", "assistant.py"))
@@ -129,19 +88,13 @@ def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator:
     # should have included only the relevant files of the package to install
     possible_packages = _PACKAGE_MAPPING.values() if _PACKAGE_NAME is None else [_PACKAGE_MAPPING[_PACKAGE_NAME]]
     for pkg in possible_packages:
-        pkg_path = os.path.join(_PATH_SRC, pkg)
-        pkg_setup = os.path.join(pkg_path, "__setup__.py")
+        pkg_setup = os.path.join(_PATH_SRC, pkg, "__setup__.py")
         if os.path.exists(pkg_setup):
             print(f"{pkg_setup} exists. Running `setuptools.setup`")
             setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup)
-            setup_args = setup_module._setup_args()
-            if _PACKAGE_NAME is None:
-                # we are installing a wheel, no need for MANIFEST.in things
-                setuptools.setup(**setup_args)
-            else:
-                # we are installing from source, set the correct manifest path
-                with _set_manifest_path(pkg_path, aggregate=pkg == "lightning"):
-                    setuptools.setup(**setup_args)
+            setup_module._adjust_manifest(pkg_name=pkg)
+            setup_args = setup_module._setup_args(pkg_name=pkg)
+            setup(**setup_args)
             break
     else:
         raise RuntimeError(f"Something's wrong, no package was installed. Package name: {_PACKAGE_NAME}")

src/lightning/MANIFEST.in

@@ -24,7 +24,28 @@ def _load_py_module(name: str, location: str) -> ModuleType:
 _SETUP_TOOLS = _load_py_module("setup_tools", os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py"))
 
 
-def _setup_args() -> Dict[str, Any]:
+def _adjust_manifest(**kwargs: Any) -> None:
+    # todo: consider rather aggregation of particular manifest adjustments
+    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
+    assert os.path.isfile(manifest_path)
+    with open(manifest_path) as fp:
+        lines = [ln.rstrip() for ln in fp.readlines()]
+    lines += [
+        "recursive-include src/lightning *.md",
+        "recursive-include requirements *.txt",
+        "recursive-include src/lightning/app/ui *",
+        "recursive-include src/lightning/cli/*-template *",  # Add templates as build-in
+        "include src/lightning/app/components/serve/catimage.png" + os.linesep,
+        # fixme: this is strange, this shall work with setup find package - include
+        "prune src/lightning_app",
+        "prune src/lightning_lite",
+        "prune src/pytorch_lightning",
+    ]
+    with open(manifest_path, "w") as fp:
+        fp.writelines([ln + os.linesep for ln in lines])
+
+
+def _setup_args(**kwargs: Any) -> Dict[str, Any]:
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))
     _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py"))
     _long_description = _SETUP_TOOLS.load_readme_description(

src/lightning/__setup__.py

@@ -41,7 +41,28 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _setup_args() -> Dict[str, Any]:
+def _adjust_manifest(**__: Any) -> None:
+    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
+    assert os.path.isfile(manifest_path)
+    with open(manifest_path) as fp:
+        lines = fp.readlines()
+    lines += [
+        "recursive-exclude src *.md" + os.linesep,
+        "recursive-exclude requirements *.txt" + os.linesep,
+        "recursive-include src/lightning_app *.md" + os.linesep,
+        "include src/lightning_app/components/serve/catimage.png" + os.linesep,
+        "recursive-include requirements/app *.txt" + os.linesep,
+        "recursive-include src/lightning_app/cli/*-template *" + os.linesep,  # Add templates
+    ]
+
+    # TODO: remove this once lightning-ui package is ready as a dependency
+    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
+
+    with open(manifest_path, "w") as fp:
+        fp.writelines(lines)
+
+
+def _setup_args(**__: Any) -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))

src/lightning_app/MANIFEST.in

@@ -42,7 +42,26 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _setup_args() -> Dict[str, Any]:
+def _adjust_manifest(**__: Any) -> None:
+    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
+    assert os.path.isfile(manifest_path)
+    with open(manifest_path) as fp:
+        lines = fp.readlines()
+    lines += [
+        "recursive-exclude src *.md" + os.linesep,
+        "recursive-exclude requirements *.txt" + os.linesep,
+        "recursive-include requirements/lite *.txt" + os.linesep,
+        "recursive-include src/lightning_lite *.md" + os.linesep,
+    ]
+
+    # TODO: remove this once lightning-ui package is ready as a dependency
+    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
+
+    with open(manifest_path, "w") as fp:
+        fp.writelines(lines)
+
+
+def _setup_args(**__: Any) -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))