From da572d7ccafee45f081a94f9b7cad04bd72059d3 Mon Sep 17 00:00:00 2001 From: Aniket Maurya Date: Tue, 7 Mar 2023 21:54:46 +0530 Subject: [PATCH] Refactor PL examples to `examples/pytorch/` (#16925) Co-authored-by: awaelchli --- .azure/gpu-tests-pytorch.yml | 6 +++--- .azure/hpu-tests.yml | 6 +++--- .github/ISSUE_TEMPLATE/1_bug_report.yaml | 2 +- .github/checkgroup.yml | 6 +++--- docs/source-pytorch/accelerators/hpu_intermediate.rst | 2 +- docs/source-pytorch/accelerators/ipu_basic.rst | 2 +- docs/source-pytorch/deploy/production_advanced.rst | 2 +- examples/README.md | 4 ++-- examples/{pl_basics => pytorch/basics}/README.md | 0 examples/{pl_basics => pytorch/basics}/autoencoder.py | 0 .../basics}/backbone_image_classifier.py | 0 .../{pl_basics => pytorch/basics}/profiler_example.py | 0 .../bug_report}/bug_report_model.ipynb | 0 .../bug_report}/bug_report_model.py | 0 .../domain_templates}/computer_vision_fine_tuning.py | 0 .../domain_templates}/generative_adversarial_net.py | 0 .../domain_templates}/imagenet.py | 0 .../domain_templates}/reinforce_learn_Qnet.py | 0 .../domain_templates}/reinforce_learn_ppo.py | 0 .../domain_templates}/semantic_segmentation.py | 0 examples/{pl_hpu => pytorch/hpu}/mnist_sample.py | 0 examples/{pl_hpu => pytorch/hpu}/ops_bf16_mnist.txt | 0 examples/{pl_hpu => pytorch/hpu}/ops_fp32_mnist.txt | 0 examples/{pl_ipu => pytorch/ipu}/mnist_sample.py | 0 .../servable_module}/production.py | 0 examples/run_pl_examples.sh | 8 ++++---- 26 files changed, 19 insertions(+), 19 deletions(-) rename examples/{pl_basics => pytorch/basics}/README.md (100%) rename examples/{pl_basics => pytorch/basics}/autoencoder.py (100%) rename examples/{pl_basics => pytorch/basics}/backbone_image_classifier.py (100%) rename examples/{pl_basics => pytorch/basics}/profiler_example.py (100%) rename examples/{pl_bug_report => pytorch/bug_report}/bug_report_model.ipynb (100%) rename examples/{pl_bug_report => pytorch/bug_report}/bug_report_model.py (100%) rename examples/{pl_domain_templates => pytorch/domain_templates}/computer_vision_fine_tuning.py (100%) rename examples/{pl_domain_templates => pytorch/domain_templates}/generative_adversarial_net.py (100%) rename examples/{pl_domain_templates => pytorch/domain_templates}/imagenet.py (100%) rename examples/{pl_domain_templates => pytorch/domain_templates}/reinforce_learn_Qnet.py (100%) rename examples/{pl_domain_templates => pytorch/domain_templates}/reinforce_learn_ppo.py (100%) rename examples/{pl_domain_templates => pytorch/domain_templates}/semantic_segmentation.py (100%) rename examples/{pl_hpu => pytorch/hpu}/mnist_sample.py (100%) rename examples/{pl_hpu => pytorch/hpu}/ops_bf16_mnist.txt (100%) rename examples/{pl_hpu => pytorch/hpu}/ops_fp32_mnist.txt (100%) rename examples/{pl_ipu => pytorch/ipu}/mnist_sample.py (100%) rename examples/{pl_servable_module => pytorch/servable_module}/production.py (100%) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 9e47b1d70ca2c..5eb3aa93737f1 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -23,8 +23,8 @@ pr: - ".actions/**" - ".azure/gpu-tests-pytorch.yml" - "examples/run_pl_examples.sh" - - "examples/pl_basics/backbone_image_classifier.py" - - "examples/pl_basics/autoencoder.py" + - "examples/pytorch/basics/backbone_image_classifier.py" + - "examples/pytorch/basics/autoencoder.py" - "requirements/pytorch/**" - "src/lightning/__about__.py" - "src/lightning/__init__.py" @@ -158,7 +158,7 @@ jobs: python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \ --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" - python .actions/assistant.py copy_replace_imports --source_dir="./examples/pl_basics" \ + python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \ --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" condition: eq(variables['PACKAGE_NAME'], 'pytorch') diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml index f8867704ccc55..7b0b72313ef46 100644 --- a/.azure/hpu-tests.yml +++ b/.azure/hpu-tests.yml @@ -19,7 +19,7 @@ pr: include: - ".actions/**" - ".azure/hpu-tests.yml" - - "examples/pl_hpu/mnist_sample.py" + - "examples/pytorch/hpu/mnist_sample.py" - "requirements/fabric/**" - "src/lightning/__about__.py" - "src/lightning/__init__.py" @@ -92,7 +92,7 @@ jobs: python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \ --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" - python .actions/assistant.py copy_replace_imports --source_dir="./examples/pl_hpu" \ + python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/hpu" \ --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" displayName: 'Adjust tests & examples' @@ -117,7 +117,7 @@ jobs: - bash: | export PYTHONPATH="${PYTHONPATH}:$(pwd)" - python "pl_hpu/mnist_sample.py" + python "pytorch/hpu/mnist_sample.py" workingDirectory: examples displayName: 'Testing: HPU examples' diff --git a/.github/ISSUE_TEMPLATE/1_bug_report.yaml b/.github/ISSUE_TEMPLATE/1_bug_report.yaml index 8b01eb21489bc..8c95f02b9e363 100644 --- a/.github/ISSUE_TEMPLATE/1_bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/1_bug_report.yaml @@ -35,7 +35,7 @@ body: label: How to reproduce the bug description: > Provide steps and example code here. - You can also paste a link to Google Colab (see our [Colab bug report template](https://colab.research.google.com/github/Lightning-AI/lightning/blob/master/examples/pl_bug_report/bug_report_model.ipynb)) or adapt this minimal [snippet](https://github.com/Lightning-AI/lightning/blob/master/examples/pl_bug_report/bug_report_model.py). + You can also paste a link to Google Colab (see our [Colab bug report template](https://colab.research.google.com/github/Lightning-AI/lightning/blob/master/examples/pytorch/bug_report/bug_report_model.ipynb)) or adapt this minimal [snippet](https://github.com/Lightning-AI/lightning/blob/master/examples/pytorch/bug_report/bug_report_model.py). placeholder: | ```python # Sample code to reproduce the problem diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 08fd1194055be..942b1045af190 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -46,8 +46,8 @@ subprojects: # only the azure GPU workflow runs the examples # all examples don't need to be added because they aren't used in CI, but these are - "examples/run_pl_examples.sh" - - "examples/pl_basics/backbone_image_classifier.py" - - "examples/pl_basics/autoencoder.py" + - "examples/pytorch/basics/backbone_image_classifier.py" + - "examples/pytorch/basics/autoencoder.py" - "requirements/pytorch/**" - "src/lightning/__about__.py" - "src/lightning/__init__.py" @@ -83,7 +83,7 @@ subprojects: paths: - ".actions/**" - ".azure/hpu-tests.yml" - - "examples/pl_hpu/mnist_sample.py" + - "examples/pytorch/hpu/mnist_sample.py" - "requirements/fabric/**" - "src/lightning/__about__.py" - "src/lightning/__init__.py" diff --git a/docs/source-pytorch/accelerators/hpu_intermediate.rst b/docs/source-pytorch/accelerators/hpu_intermediate.rst index 8c21f8b549f5c..4125fa1984398 100644 --- a/docs/source-pytorch/accelerators/hpu_intermediate.rst +++ b/docs/source-pytorch/accelerators/hpu_intermediate.rst @@ -45,7 +45,7 @@ This enables advanced users to provide their own BF16 and FP32 operator list ins accelerator="hpu", devices=1, # Optional Habana mixed precision params to be set - # Checkout `examples/pl_hpu/ops_bf16_mnist.txt` for the format + # Checkout `examples/pytorch/hpu/ops_bf16_mnist.txt` for the format plugins=[ HPUPrecisionPlugin( precision=16, diff --git a/docs/source-pytorch/accelerators/ipu_basic.rst b/docs/source-pytorch/accelerators/ipu_basic.rst index 805cfa1d67eab..50fa50b258f7b 100644 --- a/docs/source-pytorch/accelerators/ipu_basic.rst +++ b/docs/source-pytorch/accelerators/ipu_basic.rst @@ -63,7 +63,7 @@ Known limitations Currently there are some known limitations that are being addressed in the near future to make the experience seamless when moving from different devices. -Please see the `MNIST example `__ which displays most of the limitations and how to overcome them till they are resolved. +Please see the `MNIST example `__ which displays most of the limitations and how to overcome them till they are resolved. * ``self.log`` is not supported in the ``training_step``, ``validation_step``, ``test_step`` or ``predict_step``. This is due to the step function being traced and sent to the IPU devices. We're actively working on fixing this. * Multiple optimizers are not supported. ``training_step`` only supports returning one loss from the ``training_step`` function as a result. diff --git a/docs/source-pytorch/deploy/production_advanced.rst b/docs/source-pytorch/deploy/production_advanced.rst index 4efacc9363d12..cfd5c84df68a0 100644 --- a/docs/source-pytorch/deploy/production_advanced.rst +++ b/docs/source-pytorch/deploy/production_advanced.rst @@ -73,4 +73,4 @@ In order to do so, your LightningModule needs to subclass the :class:`~lightning Below you can find an example of how the serving of a resnet18 can be validated. -.. literalinclude:: ../../../examples/pl_servable_module/production.py +.. literalinclude:: ../../../examples/pytorch/servable_module/production.py diff --git a/examples/README.md b/examples/README.md index a2d0cfa5db802..d0aa50baa6e53 100644 --- a/examples/README.md +++ b/examples/README.md @@ -25,8 +25,8 @@ ______________________________________________________________________ In this folder, we have 2 simple examples that showcase the power of the Lightning Trainer. -- [Image Classifier](pl_basics/backbone_image_classifier.py) (trains arbitrary datasets with arbitrary backbones). -- [Autoencoder](pl_basics/autoencoder.py) +- [Image Classifier](pytorch/basics/backbone_image_classifier.py) (trains arbitrary datasets with arbitrary backbones). +- [Autoencoder](pytorch/basics/autoencoder.py) ______________________________________________________________________ diff --git a/examples/pl_basics/README.md b/examples/pytorch/basics/README.md similarity index 100% rename from examples/pl_basics/README.md rename to examples/pytorch/basics/README.md diff --git a/examples/pl_basics/autoencoder.py b/examples/pytorch/basics/autoencoder.py similarity index 100% rename from examples/pl_basics/autoencoder.py rename to examples/pytorch/basics/autoencoder.py diff --git a/examples/pl_basics/backbone_image_classifier.py b/examples/pytorch/basics/backbone_image_classifier.py similarity index 100% rename from examples/pl_basics/backbone_image_classifier.py rename to examples/pytorch/basics/backbone_image_classifier.py diff --git a/examples/pl_basics/profiler_example.py b/examples/pytorch/basics/profiler_example.py similarity index 100% rename from examples/pl_basics/profiler_example.py rename to examples/pytorch/basics/profiler_example.py diff --git a/examples/pl_bug_report/bug_report_model.ipynb b/examples/pytorch/bug_report/bug_report_model.ipynb similarity index 100% rename from examples/pl_bug_report/bug_report_model.ipynb rename to examples/pytorch/bug_report/bug_report_model.ipynb diff --git a/examples/pl_bug_report/bug_report_model.py b/examples/pytorch/bug_report/bug_report_model.py similarity index 100% rename from examples/pl_bug_report/bug_report_model.py rename to examples/pytorch/bug_report/bug_report_model.py diff --git a/examples/pl_domain_templates/computer_vision_fine_tuning.py b/examples/pytorch/domain_templates/computer_vision_fine_tuning.py similarity index 100% rename from examples/pl_domain_templates/computer_vision_fine_tuning.py rename to examples/pytorch/domain_templates/computer_vision_fine_tuning.py diff --git a/examples/pl_domain_templates/generative_adversarial_net.py b/examples/pytorch/domain_templates/generative_adversarial_net.py similarity index 100% rename from examples/pl_domain_templates/generative_adversarial_net.py rename to examples/pytorch/domain_templates/generative_adversarial_net.py diff --git a/examples/pl_domain_templates/imagenet.py b/examples/pytorch/domain_templates/imagenet.py similarity index 100% rename from examples/pl_domain_templates/imagenet.py rename to examples/pytorch/domain_templates/imagenet.py diff --git a/examples/pl_domain_templates/reinforce_learn_Qnet.py b/examples/pytorch/domain_templates/reinforce_learn_Qnet.py similarity index 100% rename from examples/pl_domain_templates/reinforce_learn_Qnet.py rename to examples/pytorch/domain_templates/reinforce_learn_Qnet.py diff --git a/examples/pl_domain_templates/reinforce_learn_ppo.py b/examples/pytorch/domain_templates/reinforce_learn_ppo.py similarity index 100% rename from examples/pl_domain_templates/reinforce_learn_ppo.py rename to examples/pytorch/domain_templates/reinforce_learn_ppo.py diff --git a/examples/pl_domain_templates/semantic_segmentation.py b/examples/pytorch/domain_templates/semantic_segmentation.py similarity index 100% rename from examples/pl_domain_templates/semantic_segmentation.py rename to examples/pytorch/domain_templates/semantic_segmentation.py diff --git a/examples/pl_hpu/mnist_sample.py b/examples/pytorch/hpu/mnist_sample.py similarity index 100% rename from examples/pl_hpu/mnist_sample.py rename to examples/pytorch/hpu/mnist_sample.py diff --git a/examples/pl_hpu/ops_bf16_mnist.txt b/examples/pytorch/hpu/ops_bf16_mnist.txt similarity index 100% rename from examples/pl_hpu/ops_bf16_mnist.txt rename to examples/pytorch/hpu/ops_bf16_mnist.txt diff --git a/examples/pl_hpu/ops_fp32_mnist.txt b/examples/pytorch/hpu/ops_fp32_mnist.txt similarity index 100% rename from examples/pl_hpu/ops_fp32_mnist.txt rename to examples/pytorch/hpu/ops_fp32_mnist.txt diff --git a/examples/pl_ipu/mnist_sample.py b/examples/pytorch/ipu/mnist_sample.py similarity index 100% rename from examples/pl_ipu/mnist_sample.py rename to examples/pytorch/ipu/mnist_sample.py diff --git a/examples/pl_servable_module/production.py b/examples/pytorch/servable_module/production.py similarity index 100% rename from examples/pl_servable_module/production.py rename to examples/pytorch/servable_module/production.py diff --git a/examples/run_pl_examples.sh b/examples/run_pl_examples.sh index 514129f8b5cf9..0d5410d71e8b8 100644 --- a/examples/run_pl_examples.sh +++ b/examples/run_pl_examples.sh @@ -26,11 +26,11 @@ args=" --optimizer=Adam " -python "${dir_path}/pl_basics/backbone_image_classifier.py" ${args} "$@" -python "${dir_path}/pl_basics/autoencoder.py" ${args} "$@" +python "${dir_path}/pytorch/basics/backbone_image_classifier.py" ${args} "$@" +python "${dir_path}/pytorch/basics/autoencoder.py" ${args} "$@" # test that a user can manually launch individual processes args="--trainer.devices 2 --trainer.strategy ddp --trainer.max_epochs=1 --trainer.limit_train_batches=1 --trainer.limit_val_batches=1 --trainer.limit_test_batches=1" -MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=1 python "${dir_path}/pl_basics/autoencoder.py" ${args} & -MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=0 python "${dir_path}/pl_basics/autoencoder.py" ${args} +MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=1 python "${dir_path}/pytorch/basics/autoencoder.py" ${args} & +MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=0 python "${dir_path}/pytorch/basics/autoencoder.py" ${args}