diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml index 6680f945d589d..a8da0979592a4 100644 --- a/.github/actions/pkg-check/action.yml +++ b/.github/actions/pkg-check/action.yml @@ -33,9 +33,9 @@ runs: - name: Check package run: | + git diff --exit-code || exit $? # make sure there are no local unstaged changes ls -l dist/ twine check dist/* - # python setup.py clean shell: bash - name: Unzip packages diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 4169bc33e4632..da9c4a93cc42f 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -75,7 +75,6 @@ jobs: if: ${{ matrix.pkg-name == 'app' }} run: | pip install -e . -U -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi - git checkout -- . - name: Adjust docs refs if: ${{ matrix.pkg-name == 'app' }} diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 10af40c3dd1cf..0000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -exclude *.toml # project config -exclude requirements.txt -exclude __pycache__ -include .actions/setup_tools.py -include .actions/assistant.py -include *.cff # citation info diff --git a/setup.py b/setup.py index 83a76b134c54b..00d8f406dca56 100755 --- a/setup.py +++ b/setup.py @@ -39,11 +39,15 @@ b) with a parameterization build desired packages in to standard `dist/` folder c) validate packages and publish to PyPI """ +import contextlib import os +import tempfile from importlib.util import module_from_spec, spec_from_file_location from types import ModuleType +from typing import Generator, Optional -from setuptools import setup +import setuptools +import setuptools.command.egg_info _PACKAGE_NAME = os.environ.get("PACKAGE_NAME") _PACKAGE_MAPPING = { @@ -69,6 +73,43 @@ def _load_py_module(name: str, location: str) -> ModuleType: return py +def _named_temporary_file(directory: Optional[str] = None) -> str: + # `tempfile.NamedTemporaryFile` has issues in Windows + # https://github.com/deepchem/deepchem/issues/707#issuecomment-556002823 + if directory is None: + directory = tempfile.gettempdir() + return os.path.join(directory, os.urandom(24).hex()) + + +@contextlib.contextmanager +def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator: + if aggregate: + # aggregate all MANIFEST.in contents into a single temporary file + manifest_path = _named_temporary_file(manifest_dir) + mapping = _PACKAGE_MAPPING.copy() + del mapping["lightning"] + lines = [] + for pkg in mapping.values(): + with open(os.path.join(_PATH_SRC, pkg, "MANIFEST.in")) as fh: + lines.extend(fh.readlines()) + # convert lightning_foo to lightning/foo + for new, old in mapping.items(): + lines = [line.replace(old, f"lightning/{new}") for line in lines] + with open(manifest_path, mode="w") as fp: + fp.writelines(lines) + else: + manifest_path = os.path.join(manifest_dir, "MANIFEST.in") + assert os.path.exists(manifest_path) + # avoid error: setup script specifies an absolute path + manifest_path = os.path.relpath(manifest_path, _PATH_ROOT) + setuptools.command.egg_info.manifest_maker.template = manifest_path + yield + # cleanup + setuptools.command.egg_info.manifest_maker.template = "MANIFEST.in" + if aggregate: + os.remove(manifest_path) + + if __name__ == "__main__": setup_tools = _load_py_module(name="setup_tools", location=os.path.join(_PATH_ROOT, ".actions", "setup_tools.py")) assistant = _load_py_module(name="assistant", location=os.path.join(_PATH_ROOT, ".actions", "assistant.py")) @@ -88,13 +129,19 @@ def _load_py_module(name: str, location: str) -> ModuleType: # should have included only the relevant files of the package to install possible_packages = _PACKAGE_MAPPING.values() if _PACKAGE_NAME is None else [_PACKAGE_MAPPING[_PACKAGE_NAME]] for pkg in possible_packages: - pkg_setup = os.path.join(_PATH_SRC, pkg, "__setup__.py") + pkg_path = os.path.join(_PATH_SRC, pkg) + pkg_setup = os.path.join(pkg_path, "__setup__.py") if os.path.exists(pkg_setup): print(f"{pkg_setup} exists. Running `setuptools.setup`") setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup) - setup_module._adjust_manifest(pkg_name=pkg) - setup_args = setup_module._setup_args(pkg_name=pkg) - setup(**setup_args) + setup_args = setup_module._setup_args() + if _PACKAGE_NAME is None: + # we are installing a wheel, no need for MANIFEST.in things + setuptools.setup(**setup_args) + else: + # we are installing from source, set the correct manifest path + with _set_manifest_path(pkg_path, aggregate=pkg == "lightning"): + setuptools.setup(**setup_args) break else: raise RuntimeError(f"Something's wrong, no package was installed. Package name: {_PACKAGE_NAME}") diff --git a/src/lightning/MANIFEST.in b/src/lightning/MANIFEST.in new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 6f30e218ab6e5..7d56c15dcba47 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -24,28 +24,7 @@ def _load_py_module(name: str, location: str) -> ModuleType: _SETUP_TOOLS = _load_py_module("setup_tools", os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")) -def _adjust_manifest(**kwargs: Any) -> None: - # todo: consider rather aggregation of particular manifest adjustments - manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") - assert os.path.isfile(manifest_path) - with open(manifest_path) as fp: - lines = [ln.rstrip() for ln in fp.readlines()] - lines += [ - "recursive-include src/lightning *.md", - "recursive-include requirements *.txt", - "recursive-include src/lightning/app/ui *", - "recursive-include src/lightning/cli/*-template *", # Add templates as build-in - "include src/lightning/app/components/serve/catimage.png" + os.linesep, - # fixme: this is strange, this shall work with setup find package - include - "prune src/lightning_app", - "prune src/lightning_lite", - "prune src/pytorch_lightning", - ] - with open(manifest_path, "w") as fp: - fp.writelines([ln + os.linesep for ln in lines]) - - -def _setup_args(**kwargs: Any) -> Dict[str, Any]: +def _setup_args() -> Dict[str, Any]: _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py")) _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py")) _long_description = _SETUP_TOOLS.load_readme_description( diff --git a/src/lightning_app/MANIFEST.in b/src/lightning_app/MANIFEST.in new file mode 100644 index 0000000000000..4fad4ff25a317 --- /dev/null +++ b/src/lightning_app/MANIFEST.in @@ -0,0 +1,9 @@ +include src/lightning_app/CHANGELOG.md +include src/lightning_app/README.md +recursive-include requirements/app *.txt +include .actions/setup_tools.py +include .actions/assistant.py +recursive-include src/lightning_app/cli/*-template * +# TODO: remove this once lightning-ui package is ready as a dependency +recursive-include src/lightning_app/ui * +include src/lightning/app/components/serve/catimage.png diff --git a/src/lightning_app/__setup__.py b/src/lightning_app/__setup__.py index 7a649af448a05..130079afdf381 100644 --- a/src/lightning_app/__setup__.py +++ b/src/lightning_app/__setup__.py @@ -41,28 +41,7 @@ def _prepare_extras() -> Dict[str, Any]: return extras -def _adjust_manifest(**__: Any) -> None: - manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") - assert os.path.isfile(manifest_path) - with open(manifest_path) as fp: - lines = fp.readlines() - lines += [ - "recursive-exclude src *.md" + os.linesep, - "recursive-exclude requirements *.txt" + os.linesep, - "recursive-include src/lightning_app *.md" + os.linesep, - "include src/lightning_app/components/serve/catimage.png" + os.linesep, - "recursive-include requirements/app *.txt" + os.linesep, - "recursive-include src/lightning_app/cli/*-template *" + os.linesep, # Add templates - ] - - # TODO: remove this once lightning-ui package is ready as a dependency - lines += ["recursive-include src/lightning_app/ui *" + os.linesep] - - with open(manifest_path, "w") as fp: - fp.writelines(lines) - - -def _setup_args(**__: Any) -> Dict[str, Any]: +def _setup_args() -> Dict[str, Any]: _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py") _setup_tools = _load_py_module("setup_tools", _path_setup_tools) _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py")) diff --git a/src/lightning_lite/MANIFEST.in b/src/lightning_lite/MANIFEST.in new file mode 100644 index 0000000000000..118c0aa952d9a --- /dev/null +++ b/src/lightning_lite/MANIFEST.in @@ -0,0 +1,5 @@ +include src/lightning_lite/CHANGELOG.md +include src/lightning_lite/README.md +recursive-include requirements/lite *.txt +include .actions/setup_tools.py +include .actions/assistant.py diff --git a/src/lightning_lite/__setup__.py b/src/lightning_lite/__setup__.py index b00cedb83e5b4..c0b550c37fd4c 100644 --- a/src/lightning_lite/__setup__.py +++ b/src/lightning_lite/__setup__.py @@ -42,26 +42,7 @@ def _prepare_extras() -> Dict[str, Any]: return extras -def _adjust_manifest(**__: Any) -> None: - manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") - assert os.path.isfile(manifest_path) - with open(manifest_path) as fp: - lines = fp.readlines() - lines += [ - "recursive-exclude src *.md" + os.linesep, - "recursive-exclude requirements *.txt" + os.linesep, - "recursive-include requirements/lite *.txt" + os.linesep, - "recursive-include src/lightning_lite *.md" + os.linesep, - ] - - # TODO: remove this once lightning-ui package is ready as a dependency - lines += ["recursive-include src/lightning_app/ui *" + os.linesep] - - with open(manifest_path, "w") as fp: - fp.writelines(lines) - - -def _setup_args(**__: Any) -> Dict[str, Any]: +def _setup_args() -> Dict[str, Any]: _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py") _setup_tools = _load_py_module("setup_tools", _path_setup_tools) _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py")) diff --git a/src/pytorch_lightning/MANIFEST.in b/src/pytorch_lightning/MANIFEST.in new file mode 100644 index 0000000000000..5ff4ceab4c089 --- /dev/null +++ b/src/pytorch_lightning/MANIFEST.in @@ -0,0 +1,10 @@ +# distribute the lite source code inside PL +include src/lightning_lite/CHANGELOG.md +recursive-include requirements/lite *.txt +include src/pytorch_lightning/CHANGELOG.md +include src/pytorch_lightning/README.md +recursive-include requirements/pytorch *.txt +include .actions/setup_tools.py +include .actions/assistant.py +include *.cff # citation info +include src/pytorch_lightning/py.typed # marker file for PEP 561 diff --git a/src/pytorch_lightning/__setup__.py b/src/pytorch_lightning/__setup__.py index 442bda630b884..3eea21cf0c7b0 100644 --- a/src/pytorch_lightning/__setup__.py +++ b/src/pytorch_lightning/__setup__.py @@ -44,25 +44,7 @@ def _prepare_extras() -> Dict[str, Any]: return extras -def _adjust_manifest(**__: Any) -> None: - manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") - assert os.path.isfile(manifest_path) - with open(manifest_path) as fp: - lines = fp.readlines() - lines += [ - "recursive-exclude src *.md" + os.linesep, - "recursive-exclude requirements *.txt" + os.linesep, - "recursive-include requirements/lite *.txt" + os.linesep, - "recursive-include src/lightning_lite *.md" + os.linesep, - "recursive-include src/pytorch_lightning *.md" + os.linesep, - "recursive-include requirements/pytorch *.txt" + os.linesep, - "include src/pytorch_lightning/py.typed" + os.linesep, # marker file for PEP 561 - ] - with open(manifest_path, "w") as fp: - fp.writelines(lines) - - -def _setup_args(**__: Any) -> Dict[str, Any]: +def _setup_args() -> Dict[str, Any]: _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py") _setup_tools = _load_py_module("setup_tools", _path_setup_tools) _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))