Weekly minor patch release 2.0.7 [rebase & merge]

.actions/assistant.py

@@ -81,6 +81,7 @@ def adjust(self, unfreeze: str) -> str:
         'arrow>=1.2.0'
         >>> _RequirementWithComment("arrow").adjust("major")
         'arrow'
+
         """
         out = str(self)
         if self.strict:
@@ -110,6 +111,7 @@ def _parse_requirements(strs: Union[str, Iterable[str]]) -> Iterator[_Requiremen
     >>> txt = '\\n'.join(txt)
     >>> [r.adjust('none') for r in _parse_requirements(txt)]
     ['this', 'example', 'foo  # strict', 'thing']
+
     """
     lines = yield_lines(strs)
     pip_argument = None
@@ -144,6 +146,7 @@ def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
     >>> load_requirements(path_req, "docs.txt", unfreeze="major")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     ['sphinx<...]
+
     """
     assert unfreeze in {"none", "major", "all"}
     path = Path(path_dir) / file_name
@@ -157,6 +160,7 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
 
     >>> load_readme_description(_PROJECT_ROOT, "", "")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     '...PyTorch Lightning is just organized PyTorch...'
+
     """
     path_readme = os.path.join(path_dir, "README.md")
     with open(path_readme, encoding="utf-8") as fo:
@@ -187,14 +191,13 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
 
     skip_begin = r"<!-- following section will be skipped from PyPI description -->"
     skip_end = r"<!-- end skipping PyPI description -->"
-    # todo: wrap content as commented description
-    text = re.sub(rf"{skip_begin}.+?{skip_end}", "<!--  -->", text, flags=re.IGNORECASE + re.DOTALL)
 
     # # https://github.com/Borda/pytorch-lightning/releases/download/1.1.0a6/codecov_badge.png
     # github_release_url = os.path.join(homepage, "releases", "download", version)
     # # download badge and replace url with local file
     # text = _parse_for_badge(text, github_release_url)
-    return text
+    # todo: wrap content as commented description
+    return re.sub(rf"{skip_begin}.+?{skip_end}", "<!--  -->", text, flags=re.IGNORECASE + re.DOTALL)
 
 
 def distribute_version(src_folder: str, ver_file: str = "version.info") -> None:
@@ -237,6 +240,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
     """Load all base requirements from all particular packages and prune duplicates.
 
     >>> _load_aggregate_requirements(os.path.join(_PROJECT_ROOT, "requirements"))
+
     """
     requires = [
         load_requirements(d, unfreeze="none" if freeze_requirements else "major")
@@ -293,6 +297,7 @@ def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]], lightning
      'http://pytorch_lightning.ai', \
      'from lightning_fabric import __version__', \
      '@lightning.ai']
+
     """
     out = lines[:]
     for source_import, target_import in mapping:

.azure/app-cloud-e2e.yml

@@ -60,7 +60,7 @@ jobs:
     pool: "azure-cpus"
     container:
       # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags
-      image: mcr.microsoft.com/playwright/python:v1.32.1-focal
+      image: mcr.microsoft.com/playwright/python:v1.36.0-focal
       options: "--shm-size=4gb"
     strategy:
       matrix:

.azure/gpu-tests-fabric.yml

@@ -5,8 +5,7 @@
 
 trigger:
   tags:
-    include:
-      - '*'
+    include: ['*']
   branches:
     include:
       - "master"

.azure/gpu-tests-pytorch.yml

@@ -5,8 +5,7 @@
 
 trigger:
   tags:
-    include:
-      - '*'
+    include: ['*']
   branches:
     include:
       - "master"

.github/CODEOWNERS

@@ -8,12 +8,12 @@
 * @lantiga @borda @tchaton @awaelchli
 
 # CI/CD and configs
-/.actions/      @borda @carmocca @akihironitta
-/.github/       @borda @carmocca @akihironitta
-/.azure/        @borda @carmocca @akihironitta
+/.actions/                  @borda @carmocca @ethanwharris @justusschock
+/.github/                   @borda @carmocca @ethanwharris @justusschock
+/.azure/                    @borda @carmocca @ethanwharris @justusschock
 /.azure/app-cloud-e2e.yml   @awaelchli @ethanwharris @lantiga
-/dockers/       @borda @carmocca @akihironitta
-*.yml           @borda @carmocca @akihironitta
+/dockers/                   @borda @carmocca @ethanwharris @justusschock
+*.yml                       @borda @carmocca @ethanwharris @justusschock
 
 # Docs
 /docs/                                      @edenlightning @lantiga @borda @awaelchli @RobertLaurella
@@ -24,15 +24,15 @@
 /docs/source-fabric/index.rst               @awaelchli @lantiga @carmocca
 /docs/source-pytorch/conf.py                @borda @awaelchli @carmocca
 /docs/source-pytorch/index.rst              @williamfalcon @lantiga
-/docs/source-pytorch/levels                 @williamfalcon @RobertLaurella
-/docs/source-app/                           @williamfalcon @RobertLaurella @tchaton @awaelchli
+/docs/source-pytorch/levels                 @williamfalcon @lantiga
+/docs/source-app/                           @williamfalcon @lantiga @tchaton @awaelchli
 /docs/source-app/index.rst                  @williamfalcon @lantiga
-/docs/source-app/expertise_levels           @williamfalcon @RobertLaurella
+/docs/source-app/expertise_levels           @williamfalcon @lantiga
 
 # PyTorch Lightning
 /src/lightning/pytorch                      @williamfalcon @awaelchli @carmocca @justusschock
 /src/pytorch_lightning                      @williamfalcon @awaelchli @carmocca @justusschock
-/tests/tests_pytorch                        @awaelchli @carmocca @justusschock
+/tests/tests_pytorch                        @awaelchli @carmocca @justusschock @borda
 
 # Core APIs
 /src/lightning/pytorch/callbacks/callback.py @williamfalcon @awaelchli @carmocca
@@ -41,17 +41,22 @@
 /src/lightning/pytorch/core/hooks.py         @williamfalcon @tchaton @awaelchli @carmocca
 /src/lightning/pytorch/core/module.py        @williamfalcon @tchaton @awaelchli @carmocca
 
+# Data Utilities
+/examples/data/           @nohalon @justusschock @lantiga
+/src/lightning/data/      @nohalon @justusschock @lantiga
+/tests/tests_data         @nohalon @justusschock @lantiga
+
 # Lightning Fabric
 /src/lightning/fabric                       @awaelchli @carmocca @justusschock
 /src/lightning_fabric                       @awaelchli @carmocca @justusschock
 /tests/tests_fabric                         @awaelchli @carmocca @justusschock
 
 # Lightning App
-/src/lightning/app                          @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
-/src/lightning_app                          @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
-/tests/tests_app                            @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
-/tests/integrations_app                     @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
-/examples/app_*                             @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
+/src/lightning/app                          @tchaton @lantiga @awaelchli @ethanwharris
+/src/lightning_app                          @tchaton @lantiga @awaelchli @ethanwharris
+/tests/tests_app                            @tchaton @lantiga @awaelchli @ethanwharris
+/tests/integrations_app                     @tchaton @lantiga @awaelchli @ethanwharris
+/examples/app_*                             @tchaton @lantiga @awaelchli @ethanwharris
 
 /.github/CODEOWNERS                  @williamfalcon
 /SECURITY.md                         @williamfalcon @lantiga
@@ -60,4 +65,4 @@
 /src/pytorch_lightning/__about__.py  @williamfalcon @borda @carmocca
 /src/lightning_app/__about__.py      @williamfalcon @lantiga @borda
 /src/lightning_fabric/__about__.py   @williamfalcon @borda @awaelchli
-/src/*/__setup__.py                  @borda @carmocca
+/src/*/__setup__.py                  @borda @carmocca @justusschock

.github/checkgroup.yml

@@ -66,7 +66,8 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "pytorch-lightning (GPUs)"
+      - "pytorch-lightning (GPUs) (testing Lightning | latest)"
+      - "pytorch-lightning (GPUs) (testing PyTorch | latest)"
 
   - id: "pytorch_lightning: Benchmarks"
     paths:
@@ -257,7 +258,8 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "lightning-fabric (GPUs)"
+      - "lightning-fabric (GPUs) (testing Fabric | latest)"
+      - "lightning-fabric (GPUs) (testing Lightning | latest)"
 
   # SECTION: lightning_app
 

.github/workflows/_flagship-apps.yml

@@ -29,7 +29,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: mcr.microsoft.com/playwright/python:v1.35.0-focal
+      image: mcr.microsoft.com/playwright/python:v1.36.0-focal
     strategy:
       fail-fast: false
       matrix:

.github/workflows/_legacy-checkpoints.yml

@@ -108,6 +108,7 @@ jobs:
         retention-days: ${{ env.KEEP_DAYS }}
 
     - name: Install AWS CLI
+      continue-on-error: true
       run: |
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscli.zip"
           unzip -qq awscli.zip

.github/workflows/docs-build.yml

@@ -35,9 +35,7 @@ env:
 
 jobs:
   make-doctest:
-    runs-on: ubuntu-latest
-    container:
-      image: pytorchlightning/pytorch_lightning:docs
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -47,9 +45,13 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
 
       - name: Install AWS CLI
         if: ${{ matrix.pkg-name != 'pytorch' }}
+        continue-on-error: true
         run: |
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscli.zip"
           unzip -qq awscli.zip
@@ -68,6 +70,11 @@ jobs:
           path: ${{ env.PYPI_CACHE_DIR }}
           key: pypi_wheels
 
+      - name: Install pandoc & texlive
+        if: ${{ matrix.pkg-name == 'pytorch' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc texlive-latex-extra dvipng texlive-pictures
       - name: Install package & dependencies
         run: |
           mkdir -p $PYPI_CACHE_DIR  # in case cache was not hit
@@ -87,9 +94,7 @@ jobs:
 
 
   make-html:
-    runs-on: ubuntu-latest
-    container:
-      image: pytorchlightning/pytorch_lightning:docs
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -99,9 +104,13 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
 
       - name: Install AWS CLI
         if: ${{ matrix.pkg-name != 'pytorch' }}
+        continue-on-error: true
         run: |
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscli.zip"
           unzip -qq awscli.zip
@@ -120,6 +129,11 @@ jobs:
           path: ${{ env.PYPI_CACHE_DIR }}
           key: pypi_wheels
 
+      - name: Install pandoc & texlive
+        if: ${{ matrix.pkg-name == 'pytorch' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc texlive-latex-extra dvipng texlive-pictures
       - name: Install package & dependencies
         run: |
           mkdir -p $PYPI_CACHE_DIR  # in case cache was not hit
@@ -163,7 +177,7 @@ jobs:
   deploy-docs:
     needs: [make-doctest, make-html]
     if: github.repository_owner == 'Lightning-AI' && github.event_name == 'push'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:

.pre-commit-config.yaml

@@ -50,20 +50,21 @@ repos:
       - id: detect-private-key
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.8.0
     hooks:
       - id: pyupgrade
         args: [--py38-plus]
         name: Upgrade code
 
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.6.3
+    rev: v1.7.3
     hooks:
       - id: docformatter
-        args: [--in-place, --wrap-summaries=115, --wrap-descriptions=120]
+        additional_dependencies: [tomli]
+        args: ["--in-place"]
 
   - repo: https://github.com/asottile/yesqa
-    rev: v1.4.0
+    rev: v1.5.0
     hooks:
       - id: yesqa
         name: Unused noqa
@@ -89,7 +90,7 @@ repos:
         exclude: docs/source-app
 
   - repo: https://github.com/asottile/blacken-docs
-    rev: 1.13.0
+    rev: 1.14.0
     hooks:
       - id: blacken-docs
         args: [--line-length=120]
@@ -112,7 +113,7 @@ repos:
           )$
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.262'
+    rev: v0.0.282
     hooks:
       - id: ruff
         args: ["--fix"]

README.md

@@ -27,7 +27,6 @@ ______________________________________________________________________
 [![PyPI Status](https://badge.fury.io/py/pytorch-lightning.svg)](https://badge.fury.io/py/pytorch-lightning)
 [![PyPI Status](https://pepy.tech/badge/pytorch-lightning)](https://pepy.tech/project/pytorch-lightning)
 [![Conda](https://img.shields.io/conda/v/conda-forge/lightning?label=conda&color=success)](https://anaconda.org/conda-forge/lightning)
-[![DockerHub](https://img.shields.io/docker/pulls/pytorchlightning/pytorch_lightning.svg)](https://hub.docker.com/r/pytorchlightning/pytorch_lightning)
 [![codecov](https://codecov.io/gh/Lightning-AI/lightning/branch/master/graph/badge.svg?token=SmzX8mnKlA)](https://codecov.io/gh/Lightning-AI/lightning)
 
 [![Discord](https://img.shields.io/discord/1077906959069626439?style=plastic)](https://discord.gg/VptPCZkGNa)

docs/source-app/examples/file_server/app.py

@@ -23,6 +23,7 @@ def __init__(
             drive: The drive can share data inside your application.
             base_dir: The local directory where the data will be stored.
             chunk_size: The quantity of bytes to download/upload at once.
+
         """
         super().__init__(
             cloud_build_config=L.BuildConfig(["flask, flask-cors"]),
@@ -238,4 +239,5 @@ def test_file_server_in_cloud():
 
         # 2. By calling logs = get_logs_fn(),
         # you get all the logs currently on the admin page.
+
         """

docs/source-app/examples/github_repo_runner/app.py

@@ -36,6 +36,7 @@ def __init__(
             script_args: The arguments to be provided to the script.
             requirements: The python requirements tp run the script.
             cloud_compute: The object to select the cloud instance.
+
         """
         super().__init__(
             script_path=script_path,

docs/source-app/examples/model_server_app/locust_component.py

@@ -10,6 +10,7 @@ def __init__(self, num_users: int = 100):
 
         Arguments:
             num_users: Number of users emulated by Locust
+
         """
         # Note: Using the default port 8089 of Locust.
         super().__init__(

docs/source-app/examples/model_server_app/model_server.py

@@ -18,6 +18,7 @@ class MLServer(LightningWork):
             Example: "mlserver_sklearn.SKLearnModel".
             Learn more here: $ML_SERVER_URL/tree/master/runtimes
         workers: Number of server worker.
+
     """
 
     def __init__(
@@ -51,6 +52,7 @@ def run(self, model_path: Path):
 
         Arguments:
             model_path: The path to the trained model.
+
         """
         # 1: Use the host and port at runtime so it works in the cloud.
         # $ML_SERVER_URL/blob/master/mlserver/settings.py#L50

docs/source-app/levels/basic/scripts/toy_app_1_component_pdb.py

@@ -1,10 +1,11 @@
 # app.py
 import lightning as L
+from lightning.app.pdb import set_trace
 
 class Component(L.LightningWork):
     def run(self, x):
         print(x)
-        L.pdb.set_trace()
+        set_trace()
 
 class WorkflowOrchestrator(L.LightningFlow):
     def __init__(self) -> None: