Merge branch 'master' into carmocca/mypy-1.0

.azure/gpu-tests-pytorch.yml

@@ -23,8 +23,8 @@ pr:
       - ".actions/**"
       - ".azure/gpu-tests-pytorch.yml"
       - "examples/run_pl_examples.sh"
-      - "examples/pl_basics/backbone_image_classifier.py"
-      - "examples/pl_basics/autoencoder.py"
+      - "examples/pytorch/basics/backbone_image_classifier.py"
+      - "examples/pytorch/basics/autoencoder.py"
       - "requirements/pytorch/**"
       - "src/lightning/__about__.py"
       - "src/lightning/__init__.py"
@@ -158,7 +158,7 @@ jobs:
         python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
           --source_import="lightning.fabric,lightning.pytorch" \
           --target_import="lightning_fabric,pytorch_lightning"
-        python .actions/assistant.py copy_replace_imports --source_dir="./examples/pl_basics" \
+        python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
           --source_import="lightning.fabric,lightning.pytorch" \
           --target_import="lightning_fabric,pytorch_lightning"
       condition: eq(variables['PACKAGE_NAME'], 'pytorch')

.azure/hpu-tests.yml

@@ -19,7 +19,7 @@ pr:
     include:
       - ".actions/**"
       - ".azure/hpu-tests.yml"
-      - "examples/pl_hpu/mnist_sample.py"
+      - "examples/pytorch/hpu/mnist_sample.py"
       - "requirements/fabric/**"
       - "src/lightning/__about__.py"
       - "src/lightning/__init__.py"
@@ -92,7 +92,7 @@ jobs:
         python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
           --source_import="lightning.fabric,lightning.pytorch" \
           --target_import="lightning_fabric,pytorch_lightning"
-        python .actions/assistant.py copy_replace_imports --source_dir="./examples/pl_hpu" \
+        python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/hpu" \
           --source_import="lightning.fabric,lightning.pytorch" \
           --target_import="lightning_fabric,pytorch_lightning"
       displayName: 'Adjust tests & examples'
@@ -117,7 +117,7 @@ jobs:
 
     - bash: |
         export PYTHONPATH="${PYTHONPATH}:$(pwd)"
-        python "pl_hpu/mnist_sample.py"
+        python "pytorch/hpu/mnist_sample.py"
       workingDirectory: examples
       displayName: 'Testing: HPU examples'
 

.github/ISSUE_TEMPLATE/1_bug_report.yaml

@@ -35,7 +35,7 @@ body:
       label: How to reproduce the bug
       description: >
         Provide steps and example code here.
-        You can also paste a link to Google Colab (see our [Colab bug report template](https://colab.research.google.com/github/Lightning-AI/lightning/blob/master/examples/pl_bug_report/bug_report_model.ipynb)) or adapt this minimal [snippet](https://github.com/Lightning-AI/lightning/blob/master/examples/pl_bug_report/bug_report_model.py).
+        You can also paste a link to Google Colab (see our [Colab bug report template](https://colab.research.google.com/github/Lightning-AI/lightning/blob/master/examples/pytorch/bug_report/bug_report_model.ipynb)) or adapt this minimal [snippet](https://github.com/Lightning-AI/lightning/blob/master/examples/pytorch/bug_report/bug_report_model.py).
       placeholder: |
         ```python
         # Sample code to reproduce the problem

.github/checkgroup.yml

@@ -46,8 +46,8 @@ subprojects:
       # only the azure GPU workflow runs the examples
       # all examples don't need to be added because they aren't used in CI, but these are
       - "examples/run_pl_examples.sh"
-      - "examples/pl_basics/backbone_image_classifier.py"
-      - "examples/pl_basics/autoencoder.py"
+      - "examples/pytorch/basics/backbone_image_classifier.py"
+      - "examples/pytorch/basics/autoencoder.py"
       - "requirements/pytorch/**"
       - "src/lightning/__about__.py"
       - "src/lightning/__init__.py"
@@ -83,7 +83,7 @@ subprojects:
     paths:
       - ".actions/**"
       - ".azure/hpu-tests.yml"
-      - "examples/pl_hpu/mnist_sample.py"
+      - "examples/pytorch/hpu/mnist_sample.py"
       - "requirements/fabric/**"
       - "src/lightning/__about__.py"
       - "src/lightning/__init__.py"
@@ -183,7 +183,7 @@ subprojects:
   - id: "pytorch_lightning: Docker"
     paths:
       - ".actions/**"
-      - ".github/workflows/ci-pytorch-dockers.yml"
+      - ".github/workflows/ci-dockers-pytorch.yml"
       - "dockers/**"
       - "requirements/pytorch/**"
       - "requirements/fabric/**"
@@ -196,6 +196,7 @@ subprojects:
       - "build-cuda (3.9, 1.11, 11.3.1)"
       - "build-cuda (3.9, 1.12, 11.6.1)"
       - "build-cuda (3.9, 1.13, 11.7.1)"
+      - "build-cuda (3.10, 1.13, 11.7.1)"
       - "build-hpu (1.5.0, 1.11.0)"
       - "build-ipu (3.9, 1.13)"
       - "build-NGC"

.github/workflows/docs-build.yml

@@ -165,19 +165,16 @@ jobs:
           project_id: ${{ secrets.GCS_PROJECT }}
 
       # Uploading docs to GCS, so they can be served on lightning.ai
-      - name: Upload docs/stable to GCS 🪣
+      - name: Upload docs/${{ matrix.pkg-name }}/stable to GCS 🪣
         if: startsWith(github.ref, 'refs/heads/release/')
-        # todo: consider nuke the bucket content before upload
         run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-${{ matrix.pkg-name }}/stable
 
       # Uploading docs to GCS, so they can be served on lightning.ai
-      - name: Upload docs/latest to GCS 🪣
+      - name: Upload docs/${{ matrix.pkg-name }}/latest to GCS 🪣
         if: github.ref == 'refs/heads/master'
-        # todo: consider nuke the bucket content before upload
         run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-${{ matrix.pkg-name }}/latest
 
       # Uploading docs to GCS, so they can be served on lightning.ai
-      - name: Upload docs/latest to GCS 🪣
+      - name: Upload docs/${{ matrix.pkg-name }}/release to GCS 🪣
         if: startsWith(github.ref, 'refs/tags/')
-        # todo: consider nuke the bucket content before upload
         run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-${{ matrix.pkg-name }}/${{  github.ref_name }}"

.gitignore

@@ -191,6 +191,10 @@ data/
 .lightning
 node_modules/
 
+# examples
+**/events.out.tfevents.*
+examples/**/*.png
+
 # instalation artifacts
 requirements/base.txt
 

Makefile

@@ -16,9 +16,7 @@ clean:
 	rm -rf .mypy_cache
 	rm -rf .pytest_cache
 	rm -rf ./docs/build
-	rm -rf ./docs/source-fabric/generated
-	rm -rf ./docs/source-fabric/*/generated
-	rm -rf ./docs/source-fabric/api
+	rm -rf ./docs/source-fabric/api/generated
 	rm -rf ./docs/source-pytorch/notebooks
 	rm -rf ./docs/source-pytorch/generated
 	rm -rf ./docs/source-pytorch/*/generated

docs/source-fabric/_templates/theme_variables.jinja

@@ -2,17 +2,14 @@
   'github': 'https://github.com/Lightning-AI/lightning',
   'github_issues': 'https://github.com/Lightning-AI/lightning/issues',
   'contributing': 'https://github.com/Lightning-AI/lightning/blob/master/.github/CONTRIBUTING.md',
-  'governance': 'https://github.com/Lightning-AI/lightning/blob/master/docs/source-pytorch/governance.rst',
-  'docs': 'https://lightning.rtfd.io/en/latest',
-  'twitter': 'https://twitter.com/PyTorchLightnin',
-  'discuss': 'https://pytorch-lightning.slack.com',
-  'tutorials': 'https://pt-lightning.readthedocs.io/en/latest/#tutorials',
-  'previous_pytorch_versions': 'https://pt-lightning.rtfd.io/en/latest/',
-  'home': 'https://lightning.ai/',
-  'get_started': 'https://pt-lightning.readthedocs.io/en/latest/introduction_guide.html',
-  'features': 'https://pt-lightning.rtfd.io/en/latest/',
-  'blog': 'https://www.pytorchlightning.ai/blog',
-  'resources': 'https://pt-lightning.readthedocs.io/en/latest/#community-examples',
-  'support': 'https://pt-lightning.rtfd.io/en/latest/',
+  'governance': 'https://pytorch-lightning.readthedocs.io/en/latest/governance.html',
+  'docs': 'https://lightning.ai/docs/fabric/',
+  'twitter': 'https://twitter.com/LightningAI',
+  'home': 'https://lightning.ai/docs/fabric/',
+  'get_started': '',
+  'blog': 'https://lightning.ai/pages/blog/',
+  'support': '',
+  'community': 'https://www.pytorchlightning.ai/community',
+  'forums': 'https://lightning.ai/forums/',
 }
 -%}

docs/source-fabric/advanced/multiple_setup.rst

@@ -0,0 +1,116 @@
+:orphan:
+
+##############################
+Multiple Models and Optimizers
+##############################
+
+Fabric makes it very easy to work with multiple models and/or optimizers at once in your training workflow.
+Examples of where this comes in handy are Generative Adversarial Networks (GANs), Auto-encoders, meta-learning and more.
+
+
+----
+
+************************
+One model, one optimizer
+************************
+
+Fabric has a simple guideline you should follow:
+If you have an optimizer, you should set it up together with the model to make your code truly strategy-agnostic.
+
+.. code-block:: python
+
+    import torch
+    from lightning.fabric import Fabric
+
+    fabric = Fabric()
+
+    # Instantiate model and optimizer
+    model = LitModel()
+    optimizer = torch.optim.Adam(model.parameters())
+
+    # Set up the model and optimizer together
+    model, optimizer = fabric.setup(model, optimizer)
+
+
+Depending on the selected strategy, the :meth:`~lightning.fabric.fabric.Fabric.setup` method will wrap and link the model with the optimizer.
+
+
+----
+
+
+******************************
+One model, multiple optimizers
+******************************
+
+You can also have multiple optimizers over a single model.
+This is useful if you need specific optimizers or learning rates for parts of the model.
+
+.. code-block:: python
+
+    # Instantiate model and optimizers
+    model = LitModel()
+    optimizer1 = torch.optim.SGD(model.layer1.parameters(), lr=0.003)
+    optimizer2 = torch.optim.SGD(model.layer2.parameters(), lr=0.01)
+
+    # Set up the model and optimizers together
+    model, optimizer1, optimizer2 = fabric.setup(model, optimizer1, optimizer2)
+
+
+
+----
+
+
+******************************
+Multiple models, one optimizer
+******************************
+
+Using a single optimizer to update multiple models is possible too.
+The best way to do this is to group all your individual models under one top level ``nn.Module``:
+
+.. code-block:: python
+
+    class AutoEncoder(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+
+            # Group all models under a common nn.Module
+            self.encoder = Encoder()
+            self.decoder = Decoder()
+
+Now all of these models can be treated as a single one:
+
+.. code-block:: python
+
+    # Instantiate the big model
+    autoencoder = AutoEncoder()
+    optimizer = ...
+
+    # Set up the model(s) and optimizer together
+    autoencoder, optimizer = fabric.setup(autoencoder, optimizer)
+
+
+----
+
+
+************************************
+Multiple models, multiple optimizers
+************************************
+
+You can pair up as many models and optimizers as you want. For example, two models with one optimizer each:
+
+.. code-block:: python
+
+    # Two models
+    generator = Generator()
+    discriminator = Discriminator()
+
+    # Two optimizers
+    optimizer_gen = torch.optim.SGD(generator.parameters(), lr=0.01)
+    optimizer_dis = torch.optim.SGD(discriminator.parameters(), lr=0.001)
+
+    # Set up generator
+    generator, optimizer_gen = fabric.setup(generator, optimizer_gen)
+    # Set up discriminator
+    discriminator, optimizer_dis = fabric.setup(discriminator, optimizer_dis)
+
+For a full example of this use case, see our `GAN example <https://github.com/Lightning-AI/lightning/blob/master/examples/fabric/dcgan>`_.

docs/source-fabric/api/utilities.rst

@@ -40,7 +40,7 @@ For example, when generating noise or data augmentations. This is very straightf
     fabric.seed_everything(seed + fabric.global_rank)
 
 
-By default, ``seed_everything`` also handles the initialization of the seed in :class:`~torch.utils.data.DataLoader` worker processes:
+By default, :meth:`~lightning.fabric.fabric.Fabric.seed_everything` also handles the initialization of the seed in :class:`~torch.utils.data.DataLoader` worker processes:
 
 .. code-block:: python
 

docs/source-fabric/examples/index.rst

@@ -46,6 +46,13 @@ Examples
     :col_css: col-md-4
     :height: 150
 
+.. displayitem::
+    :header: K-Fold Cross Validation
+    :description: Cross validation helps you estimate the generalization error of a model and select the best one.
+    :button_link: https://github.com/Lightning-AI/lightning/tree/master/examples/fabric/kfold_cv
+    :col_css: col-md-4
+    :height: 150
+
 .. displayitem::
     :header: Active Learning
     :description: Coming soon

docs/source-fabric/fundamentals/code_structure.rst

@@ -4,7 +4,7 @@ How to structure your code with Fabric
 
 Fabric is flexible enough to adapt to any project structure, regardless of whether you are experimenting with a simple script or an extensive framework, because it makes no assumptions about how your code is organized.
 Despite the ultimate freedom, this page is meant to give beginners a template for how to organize a typical training script with Fabric:
-We also have several :ref:`examples <Fabric Examples>` that you can take inspiration from.
+We also have several :doc:`examples <../examples/index>` that you can take inspiration from.
 
 
 ----
@@ -140,4 +140,29 @@ Here is how the code would be structured if we did that periodically during trai
 Full Trainer
 ************
 
-Coming soon.
+Building a fully-fledged, personalized Trainer can be a lot of work.
+To get started quickly, copy `this <https://github.com/Lightning-AI/lightning/tree/master/examples/fabric/build_your_own_trainer>`_ Trainer template and adapt it to your needs.
+
+- Only ~500 lines of code, all in one file
+- Relies on Fabric to configure accelerator, devices, strategy
+- Simple epoch based training with validation loop
+- Only essential features included: Checkpointing, loggers, progress bar, callbacks, gradient accumulation
+
+
+.. raw:: html
+
+    <div class="display-card-container">
+        <div class="row">
+
+.. displayitem::
+    :header: Trainer Template
+    :description: Take our Fabric Trainer template and customize it for your needs
+    :button_link: https://github.com/Lightning-AI/lightning/tree/master/examples/fabric/build_your_own_trainer
+    :col_css: col-md-4
+    :height: 150
+    :tag: intermediate
+
+.. raw:: html
+
+        </div>
+    </div>

docs/source-fabric/fundamentals/convert.rst

@@ -18,7 +18,7 @@ Here are five easy steps to let :class:`~lightning.fabric.fabric.Fabric` scale y
 
     fabric.launch()
 
-**Step 3:** Call :meth:`~lightning.fabric.fabric.Fabric.setup` on each model and optimizer pair and :meth:`~lightning_fabric.fabric.Fabric.setup_dataloaders` on all your data loaders.
+**Step 3:** Call :meth:`~lightning.fabric.fabric.Fabric.setup` on each model and optimizer pair and :meth:`~lightning.fabric.fabric.Fabric.setup_dataloaders` on all your data loaders.
 
 .. code-block:: python
 
@@ -90,7 +90,7 @@ All steps combined, this is how your code will change:
 
 
 That's it! You can now train on any device at any scale with a switch of a flag.
-Check out our before-and-after example for `image classification <https://github.com/Lightning-AI/lightning/blob/master/examples/fabric/image_classifier/README.md>`_ and many more :ref:`examples <Fabric Examples>` that use Fabric.
+Check out our before-and-after example for `image classification <https://github.com/Lightning-AI/lightning/blob/master/examples/fabric/image_classifier/README.md>`_ and many more :doc:`examples <../examples/index>` that use Fabric.
 
 **********
 Next steps
@@ -105,7 +105,7 @@ Next steps
     :header: Examples
     :description: See examples across computer vision, NLP, RL, etc.
     :col_css: col-md-4
-    :button_link: ../fabric.html#examples
+    :button_link: ../examples/index.html
     :height: 150
     :tag: basic
 
@@ -121,7 +121,7 @@ Next steps
     :header: Build your own Trainer
     :description: Learn how to build a trainer tailored for you
     :col_css: col-md-4
-    :button_link: ../fabric.html#build-your-own-trainer
+    :button_link: ../index.html#build-your-own-trainer
     :height: 150
     :tag: intermediate
 

docs/source-fabric/fundamentals/launch.rst

@@ -15,7 +15,7 @@ To run your code distributed across many devices and many machines, you need to
 Simple Launch
 *************
 
-You can configure and launch processes on your machine directly with Fabric's :meth:`~lightning_fabric.fabric.Fabric.launch` method:
+You can configure and launch processes on your machine directly with Fabric's :meth:`~lightning.fabric.fabric.Fabric.launch` method:
 
 .. code-block:: python
 

docs/source-fabric/fundamentals/precision.rst

@@ -13,7 +13,7 @@ Mixed precision training delivers significant computational speedup by conductin
 Switching to mixed precision has resulted in considerable training speedups since the introduction of Tensor Cores in the Volta and Turing architectures.
 It combines FP32 and lower-bit floating points (such as FP16) to reduce memory footprint and increase performance during model training and evaluation.
 It accomplishes this by recognizing the steps that require complete accuracy and employing a 32-bit floating point for those steps only while using a 16-bit floating point for the rest.
-Compared to complete precision training, mixed precision training delivers all these benefits while ensuring no task-specific accuracy is lost [`1 <https://docs.nvidia.com/deeplearning/performance/mixed-precision-training/index.html>`_].
+Compared to complete precision training, mixed precision training delivers all these benefits while ensuring no task-specific accuracy is lost `[1] <https://docs.nvidia.com/deeplearning/performance/mixed-precision-training/index.html>`_.
 
 This is how you select the precision in Fabric: