diff --git a/.actions/assistant.py b/.actions/assistant.py index 664f3e8a89e75..c1e2f08c340cb 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -14,10 +14,8 @@ import glob import logging import os -import pathlib import re import shutil -import tarfile import tempfile import urllib.request from distutils.version import LooseVersion @@ -35,11 +33,6 @@ "requirements/pytorch/strategies.txt", "requirements/pytorch/examples.txt", ), - "app": ( - "requirements/app/app.txt", - "requirements/app/cloud.txt", - "requirements/app/ui.txt", - ), "fabric": ( "requirements/fabric/base.txt", "requirements/fabric/strategies.txt", @@ -216,30 +209,6 @@ def distribute_version(src_folder: str, ver_file: str = "version.info") -> None: shutil.copy2(ver_template, fpath) -def _download_frontend(pkg_path: str, version: str = "v0.0.0"): - """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct - directory.""" - - try: - frontend_dir = pathlib.Path(pkg_path, "ui") - download_dir = tempfile.mkdtemp() - - shutil.rmtree(frontend_dir, ignore_errors=True) - # TODO: remove this once lightning-ui package is ready as a dependency - frontend_release_url = f"https://lightning-packages.s3.amazonaws.com/ui/{version}.tar.gz" - response = urllib.request.urlopen(frontend_release_url) - - file = tarfile.open(fileobj=response, mode="r|gz") - file.extractall(path=download_dir) # noqa: S202 - - shutil.move(download_dir, frontend_dir) - print("The Lightning UI has successfully been downloaded!") - - # If installing from source without internet connection, we don't want to break the installation - except Exception: - print("The Lightning UI downloading has failed!") - - def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requirements: bool = False) -> None: """Load all base requirements from all particular packages and prune duplicates. @@ -466,7 +435,7 @@ def pull_docs_files( raise RuntimeError(f"Requesting file '{zip_url}' does not exist or it is just unavailable.") with zipfile.ZipFile(zip_file, "r") as zip_ref: - zip_ref.extractall(tmp) # noqa: S202 + zip_ref.extractall(tmp) zip_dirs = [d for d in glob.glob(os.path.join(tmp, "*")) if os.path.isdir(d)] # check that the extracted archive has only repo folder diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml deleted file mode 100644 index 53174c42aba02..0000000000000 --- a/.azure/app-cloud-e2e.yml +++ /dev/null @@ -1,206 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: - tags: - include: - - "*" - branches: - include: - - "master" - - "release/*" - - "refs/tags/*" - -schedules: - - cron: "0 0 * * *" # At the end of every day - displayName: Daily midnight testing - branches: - include: - - "master" - -pr: - branches: - include: - - "master" - - "release/*" - paths: - include: - - ".actions/*" - - ".azure/app-cloud-e2e.yml" - - "src/lightning/__about__.py" - - "src/lightning/__init__.py" - - "src/lightning/__main__.py" - - "src/lightning/__setup__.py" - - "src/lightning/__version__.py" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "examples/app/**" - - "requirements/app/**" - - "tests/integrations_app/**" - - "setup.py" - exclude: - - "!tests/integrations_app/flagship/**" - - "requirements/*/docs.txt" - - "*.md" - - "**/*.md" - -# variables are automatically exported as environment variables so this will override pip's default cache dir -variables: - - name: pip_cache_dir - value: $(Pipeline.Workspace)/.pip - - name: local_id - value: $(Build.BuildId) - - name: video_artifact_dir - value: ./videos - -jobs: - - job: test_e2e - pool: "azure-cpus" - container: - # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags - image: mcr.microsoft.com/playwright/python:v1.38.0-focal - options: "--shm-size=4gb" - strategy: - matrix: - "App: v0_app": - name: "v0_app" - dir: "public" - "App: boring_app": - name: "boring_app" - dir: "public" - "App: template_streamlit_ui": - name: "template_streamlit_ui" - dir: "public" - "App: template_react_ui": - name: "template_react_ui" - dir: "public" - # 'App: template_jupyterlab': # TODO: clarify where these files lives - # name: "template_jupyterlab" - "App: installation_commands_app": - name: "installation_commands_app" - dir: "public" - "App: drive": - name: "drive" - dir: "public" - "App: payload": - name: "payload" - dir: "public" - "App: commands_and_api": - name: "commands_and_api" - dir: "public" - "App: quick_start": - name: "quick_start" - dir: "public" - "App: idle_timeout": - name: "idle_timeout" - dir: "local" - "App: collect_failures": - name: "collect_failures" - dir: "local" - "App: custom_work_dependencies": - name: "custom_work_dependencies" - dir: "local" - timeoutInMinutes: "15" - cancelTimeoutInMinutes: "1" - # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace - workspace: - clean: all - variables: - FREEZE_REQUIREMENTS: "1" - HEADLESS: "1" - PACKAGE_LIGHTNING: "1" - CLOUD: "1" - VIDEO_LOCATION: $(video_artifact_dir) - PR_NUMBER: $(local_id) - TEST_APP_NAME: $(name) - TEST_APP_FOLDER: $(dir) - HAR_LOCATION: "./artifacts/hars" - SLOW_MO: "50" - LIGHTNING_DEBUG: "1" - steps: - - script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)' - displayName: "Set id for this PR" - condition: eq(variables['Build.Reason'], 'PullRequest') - - - bash: | - whoami - mkdir -p "$(video_artifact_dir)/$(name)" - printf "local id: $(local_id)\n" - python --version - pip --version - echo "allow fail: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}" - displayName: "Info" - - # TODO: we are testing it as `lightning`, so add also version for `lightning_app` - - bash: | - pip install -e .[app-dev] \ - -f https://download.pytorch.org/whl/cpu/torch_stable.html - displayName: "Install Lightning & dependencies" - - - bash: python -m playwright install # --with-deps - displayName: "Install Playwright system dependencies" - - # The magic happens here it doesn't need to install the quick start dependencies. - # This test is very important to test the main user story of lightning app. - # It also e2e tests running on cloud without installing dependencies. - - bash: | - git clone https://github.com/Lightning-AI/lightning-quick-start examples/app/quick-start - # without succeeded this could run even if the job has already failed - condition: and(succeeded(), eq(variables['name'], 'quick_start')) - displayName: "Clone Quick start Repo" - - bash: | - git clone https://github.com/Lightning-AI/lightning-template-react examples/app/template_react_ui - # without succeeded this could run even if the job has already failed - condition: and(succeeded(), eq(variables['name'], 'template_react_ui')) - displayName: "Clone Template React UI Repo" - - # Replace imports to use `lightning` instead of `lightning_app` since we install lightning only ATM - - bash: | - pip install -q -r .actions/requirements.txt - python .actions/assistant.py copy_replace_imports \ - --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app" - displayName: "Adjust examples" - - - bash: pip --version && pip list - displayName: "List pip dependency" - - - bash: | - ls -l examples/app/$(TEST_APP_NAME) - echo ${TEST_FILE} - python -m pytest ${TEST_FILE}::test_$(TEST_APP_NAME)_example_cloud \ - --timeout=360 --capture=no -v --color=yes - env: - TEST_FILE: tests/integrations_app/$(TEST_APP_FOLDER)/test_$(TEST_APP_NAME).py - #LAI_USER: $(LAI_USER) # for STAGING - #LAI_PASS: $(LAI_PASS) # for STAGING - LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD) - LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) - LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD) - LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) - # Todo: investigate why these apps are failing - continueOnError: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }} - displayName: "Run the tests" - - - task: PublishPipelineArtifact@1 - condition: failed() - inputs: - path: "$(video_artifact_dir)/$(name)" - artifactName: $(name) - publishLocation: "pipeline" - displayName: "Publish videos" - - - bash: | - time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" - condition: always() - continueOnError: "true" - timeoutInMinutes: "3" - env: - #LAI_USER: $(LAI_USER) # for STAGING - #LAI_PASS: $(LAI_PASS) # for STAGING - LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD) - LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) - LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD) - LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) - displayName: "Clean Previous Apps" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 821543cea4438..cdc2b63b2379d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,7 +11,6 @@ /.actions/ @borda @ethanwharris @justusschock /.github/ @borda @ethanwharris @justusschock /.azure/ @borda @ethanwharris @justusschock -/.azure/app-cloud-e2e.yml @awaelchli @ethanwharris @lantiga /dockers/ @borda @ethanwharris @justusschock *.yml @borda @ethanwharris @justusschock @@ -25,7 +24,6 @@ /docs/source-pytorch/conf.py @borda @awaelchli /docs/source-pytorch/index.rst @williamfalcon @lantiga /docs/source-pytorch/levels @williamfalcon @lantiga -/docs/source-app/ @williamfalcon @lantiga @tchaton # PyTorch Lightning /src/lightning/pytorch @lantiga @borda @tchaton @awaelchli @justusschock @@ -36,18 +34,10 @@ # Lightning Fabric /src/lightning/fabric @lantiga @borda @tchaton @awaelchli @justusschock -# Lightning App -/src/lightning/app @tchaton @lantiga @awaelchli @ethanwharris -/src/lightning_app @tchaton @lantiga @awaelchli @ethanwharris -/tests/tests_app @tchaton @lantiga @awaelchli @ethanwharris -/tests/integrations_app @tchaton @lantiga @awaelchli @ethanwharris -/examples/app_* @tchaton @lantiga @awaelchli @ethanwharris - /.github/CODEOWNERS @williamfalcon /SECURITY.md @williamfalcon @lantiga /README.md @williamfalcon @lantiga /setup.py @williamfalcon @borda /src/pytorch_lightning/__about__.py @williamfalcon @borda -/src/lightning_app/__about__.py @williamfalcon @lantiga @borda /src/lightning_fabric/__about__.py @williamfalcon @borda @awaelchli /src/*/__setup__.py @borda @justusschock diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index db6543b8cb40e..775dc5dee77dc 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -310,20 +310,6 @@ and the last true master commit is `ccc111` and your first commit is `mmm222`. git push -f ``` -#### How to run an app on the cloud with a local version of lightning - -The lightning cloud uses the latest release by default. However, you might want to run your app with some local changes you've made to the lightning framework. To use your local version of lightning on the cloud, set the following environment variable: - -```bash -git clone https://github.com/Lightning-AI/lightning.git -cd lightning -pip install -e . -export PACKAGE_LIGHTNING=1 # <- this is the magic to use your version (not mainstream PyPI)! -lightning run app app.py --cloud -``` - -By setting `PACKAGE_LIGHTNING=1`, lightning packages the lightning source code in your local directory in addition to your app source code and uploads them to the cloud. - ### Bonus Workflow Tip If you don't want to remember all the commands above every time you want to push some code/setup a Lightning Dev environment on a new VM, you can set up bash aliases for some common commands. You can add these to one of your `~/.bashrc`, `~/.zshrc`, or `~/.bash_aliases` files. diff --git a/.github/ISSUE_TEMPLATE/1_bug_report.yaml b/.github/ISSUE_TEMPLATE/1_bug_report.yaml index e6037c6d3bbb0..a3e2cfabe58f9 100644 --- a/.github/ISSUE_TEMPLATE/1_bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/1_bug_report.yaml @@ -91,9 +91,7 @@ body: Current environment ``` - #- Lightning Component (e.g. Trainer, LightningModule, LightningApp, LightningWork, LightningFlow): #- PyTorch Lightning Version (e.g., 1.5.0): - #- Lightning App Version (e.g., 0.5.2): #- PyTorch Version (e.g., 2.0): #- Python version (e.g., 3.9): #- OS (e.g., Linux): diff --git a/.github/actions/pkg-install/action.yml b/.github/actions/pkg-install/action.yml index 96aab24a66a4c..e379f37aef68a 100644 --- a/.github/actions/pkg-install/action.yml +++ b/.github/actions/pkg-install/action.yml @@ -25,7 +25,7 @@ runs: run: | import os, glob - lut = {'app': 'lightning_app', 'fabric': 'lightning_fabric', 'pytorch': 'pytorch_lightning'} + lut = {'fabric': 'lightning_fabric', 'pytorch': 'pytorch_lightning'} act_pkg = lut.get('${{inputs.pkg-name}}', 'lightning') pkg_sdist = glob.glob('*.tar.gz')[0] pkg_wheel = glob.glob('*.whl')[0] diff --git a/.github/actions/prep-apps/action.yml b/.github/actions/prep-apps/action.yml deleted file mode 100644 index 2cd1655e2a36d..0000000000000 --- a/.github/actions/prep-apps/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Adjust App environment -description: make adjustment specific for selected App - -inputs: - name: - description: application name - required: true - -runs: - using: "composite" - steps: - - name: adjust env -> Flashy - if: inputs.name == 'flashy' - working-directory: tests/_flagship-app - run: | - ls -l . - pip install -r requirements-dev.txt -f $TORCH_URL - pip install -e . -f $TORCH_URL - shell: bash - - - name: adjust env -> Muse - if: inputs.name == 'muse' - working-directory: tests/ - run: | - pip install -e _flagship-app -f $TORCH_URL - cp _flagship-app/tests/test_app.py \ - integrations_app/flagship/test_${{ inputs.name }}.py - shell: bash - - - name: adjust env -> Jupyter - if: inputs.name == 'jupyter' - working-directory: tests/ - run: | - pip install -e _flagship-app -f $TORCH_URL - # pip install -r _flagship-app/tests/requirements-dev.txt - cp _flagship-app/tests/test_jupyter_app.py \ - integrations_app/flagship/test_${{ inputs.name }}.py - shell: bash diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 79b65664d2eb8..3774a56e2f480 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -213,121 +213,6 @@ subprojects: checks: - "test-on-tpus (pytorch, pjrt, v4-8)" - # SECTION: lightning_app - - - id: "lightning_app: Tests workflow" - paths: - - ".actions/*" - - ".github/workflows/ci-tests-app.yml" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "tests/tests_app/**" - - "requirements/app/**" - - "setup.py" - - "!requirements/*/docs.txt" - - "!*.md" - - "!**/*.md" - checks: - - "app-pytest (macOS-12, lightning, 3.8, latest)" - - "app-pytest (macOS-12, lightning, 3.8, oldest)" - - "app-pytest (macOS-12, app, 3.9, latest)" - - "app-pytest (macOS-12, app, 3.11, latest)" - - "app-pytest (ubuntu-20.04, lightning, 3.8, latest)" - - "app-pytest (ubuntu-20.04, lightning, 3.8, oldest)" - - "app-pytest (ubuntu-20.04, app, 3.9, latest)" - - "app-pytest (ubuntu-22.04, app, 3.11, latest)" - - "app-pytest (windows-2022, lightning, 3.8, latest)" - - "app-pytest (windows-2022, lightning, 3.8, oldest)" - - "app-pytest (windows-2022, app, 3.8, latest)" - - "app-pytest (windows-2022, app, 3.11, latest)" - - - id: "lightning_app: Examples" - paths: - - ".actions/*" - - ".github/workflows/ci-examples-app.yml" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "tests/integrations_app/**" - - "!tests/integrations_app/flagship/**" - - "examples/app/**" - - "requirements/app/**" - - "setup.py" - - "!requirements/*/docs.txt" - - "!*.md" - - "!**/*.md" - checks: - - "app-examples (macOS-12, lightning, 3.9, latest)" - - "app-examples (macOS-12, lightning, 3.9, oldest)" - - "app-examples (macOS-12, app, 3.9, latest)" - - "app-examples (ubuntu-20.04, lightning, 3.9, latest)" - - "app-examples (ubuntu-20.04, lightning, 3.9, oldest)" - - "app-examples (ubuntu-20.04, app, 3.9, latest)" - - "app-examples (windows-2022, lightning, 3.9, latest)" - - "app-examples (windows-2022, lightning, 3.9, oldest)" - - "app-examples (windows-2022, app, 3.9, latest)" - - #- id: "lightning: Flagships" - # paths: - # - ".github/workflows/_flagship-apps.yml" - # - ".github/workflows/ci-flagship-apps.yml" - # - "github/actions/prep-apps/action.yml" - # - "tests/integrations_app/flagship/**" - # checks: - # - "test-flagships / run-flagships (flashy, Lightning-Universe/Flashy-app)" - - - id: "lightning: Store" - paths: - - ".github/workflows/ci-tests-store.yml" - - "src/lightning/__init__.py" - - "src/lightning/__setup__.py" - - "src/lightning/__version__.py" - - "src/lightning/store/**" - - "tests/tests_store/**" - checks: - - "store-cpu (macOS-14, lightning, 3.10, 2.0)" - - "store-cpu (ubuntu-20.04, lightning, 3.10, 2.0)" - - "store-cpu (windows-2022, lightning, 3.10, 2.0)" - - # FixMe: re-enable when BE stabilize - # - id: "lightning_app: Azure" - # paths: - # - ".actions/*" - # - ".azure/app-cloud-e2e.yml" - # - "src/lightning/__about__.py" - # - "src/lightning/__init__.py" - # - "src/lightning/__main__.py" - # - "src/lightning/__setup__.py" - # - "src/lightning/__version__.py" - # - "src/lightning/app/**" - # - "src/lightning_app/*" - # - "examples/app/**" - # - "requirements/app/**" - # - "tests/integrations_app/**" - # - "!tests/integrations_app/flagship/**" - # - "setup.py" - # - "!requirements/*/docs.txt" - # - "!*.md" - # - "!**/*.md" - # checks: - # - "App.cloud-e2e" - - - id: "lightning_app: Docs" - paths: - - ".actions/*" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "docs/source-app/**" - - ".github/workflows/docs-build.yml" - - "requirements/docs.txt" - - "requirements/app/**" - - "setup.py" - - "pyproject.toml" # includes metadata used in the package creation - - "!*.md" - - "!**/*.md" - checks: - - "docs-make (app, doctest)" - - "docs-make (app, html)" - # SECTION: common - id: "mypy" @@ -359,8 +244,6 @@ subprojects: - "!*.md" - "!**/*.md" checks: - - "install-pkg (ubuntu-22.04, app, 3.8)" - - "install-pkg (ubuntu-22.04, app, 3.11)" - "install-pkg (ubuntu-22.04, fabric, 3.8)" - "install-pkg (ubuntu-22.04, fabric, 3.11)" - "install-pkg (ubuntu-22.04, pytorch, 3.8)" @@ -369,8 +252,6 @@ subprojects: - "install-pkg (ubuntu-22.04, lightning, 3.11)" - "install-pkg (ubuntu-22.04, notset, 3.8)" - "install-pkg (ubuntu-22.04, notset, 3.11)" - - "install-pkg (macOS-12, app, 3.8)" - - "install-pkg (macOS-12, app, 3.11)" - "install-pkg (macOS-12, fabric, 3.8)" - "install-pkg (macOS-12, fabric, 3.11)" - "install-pkg (macOS-12, pytorch, 3.8)" @@ -379,8 +260,6 @@ subprojects: - "install-pkg (macOS-12, lightning, 3.11)" - "install-pkg (macOS-12, notset, 3.8)" - "install-pkg (macOS-12, notset, 3.11)" - - "install-pkg (windows-2022, app, 3.8)" - - "install-pkg (windows-2022, app, 3.11)" - "install-pkg (windows-2022, fabric, 3.8)" - "install-pkg (windows-2022, fabric, 3.11)" - "install-pkg (windows-2022, pytorch, 3.8)" diff --git a/.github/label-change.yml b/.github/label-change.yml index 1f5b809a5fa60..8312c612b1207 100644 --- a/.github/label-change.yml +++ b/.github/label-change.yml @@ -1,28 +1,9 @@ -app: - - changed-files: - - any-glob-to-any-file: - - "src/lightning/app/**" - - "src/lightning_app/*" - - "tests/tests_app/**" - - "tests/integrations_app/**" - - "tests/integrations_app_examples/**" - - "examples/app/**" - - "docs/source-app/**" - - "requirements/app/**" - data: - changed-files: - any-glob-to-any-file: - "src/lightning/data/**" - "requirements/data/**" -store: - - changed-files: - - any-glob-to-any-file: - - "src/lightning/store/**" - - "tests/tests_store/**" - - "requirements/store/**" - pl: - changed-files: - any-glob-to-any-file: @@ -70,7 +51,6 @@ package: - "src/version.info" - "src/lightning/*/__setup__.py" - "src/lightning/*/__version__.py" - - "src/lightning_app/*" - "src/lightning_fabric/*" - "src/pytorch_lightning/*" diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0c2c5a69c4b4c..58f4afe529509 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,15 +8,12 @@ Brief description of all our automation tools used for boosting development perf | workflow file | action | accelerator | | -------------------------------------- | ----------------------------------------------------------------------------------------- | ----------- | -| .github/workflows/ci-tests-app.yml | Run all tests (may need internet connectivity). | CPU | | .github/workflows/ci-tests-fabric.yml | Run all tests except for accelerator-specific and standalone. | CPU | | .github/workflows/ci-tests-pytorch.yml | Run all tests except for accelerator-specific and standalone. | CPU | | .github/workflows/ci-tests-data.yml | Run unit and integration tests with data pipelining. | CPU | -| .github/workflows/ci-tests-store.yml | Run integration tests on uploading models to cloud. | CPU | | .azure-pipelines/gpu-tests-fabric.yml | Run only GPU-specific tests, standalone\*, and examples. | GPU | | .azure-pipelines/gpu-tests-pytorch.yml | Run only GPU-specific tests, standalone\*, and examples. | GPU | | .azure-pipelines/gpu-benchmarks.yml | Run speed/memory benchmarks for parity with vanila PyTorch. | GPU | -| .github/workflows/ci-examples-app.yml | Run integration tests with App examples. | CPU | | .github/workflows/ci-flagship-apps.yml | Run end-2-end tests with full applications, including deployment to the production cloud. | CPU | | .github/workflows/ci-tests-pytorch.yml | Run all tests except for accelerator-specific, standalone and slow tests. | CPU | | .github/workflows/tpu-tests.yml | Run only TPU-specific tests. Requires that the PR title contains '\[TPU\]' | TPU | diff --git a/.github/workflows/_build-packages.yml b/.github/workflows/_build-packages.yml index cb58031805803..e0262ac63b685 100644 --- a/.github/workflows/_build-packages.yml +++ b/.github/workflows/_build-packages.yml @@ -12,7 +12,7 @@ on: required: false type: string default: | - ["lightning", "app", "fabric", "pytorch"] + ["lightning", "fabric", "pytorch"] defaults: run: diff --git a/.github/workflows/_flagship-apps.yml b/.github/workflows/_flagship-apps.yml deleted file mode 100644 index 3282c58ababb5..0000000000000 --- a/.github/workflows/_flagship-apps.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Call integration of flagship Apps - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - workflow_call: - inputs: - environment: - description: "Lightning environment" - required: false - default: "PROD" - type: string - workflow_dispatch: - inputs: - environment: - description: "Lightning environment" - required: true - default: "PROD" - type: choice - options: - - PROD - - STAGING - -defaults: - run: - shell: bash - -jobs: - run-flagships: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/playwright/python:v1.38.0-focal - strategy: - fail-fast: false - matrix: - include: - - { app: "flashy", repo: "Lightning-Universe/Flashy-app" } - - { app: "muse", repo: "Lightning-Universe/stable-diffusion-deploy" } - - { app: "jupyter", repo: "Lightning-Universe/Jupyter-component" } - - # TODO: - # - Training Studio - # - Echo - # - StreamLit / Gradio - # - All homepage & docs apps - - env: - HEADLESS: "1" - PACKAGE_LIGHTNING: "1" - CLOUD: "1" - VIDEO_LOCATION: "./videos" - HAR_LOCATION: "./artifacts/hars" - SLOW_MO: "50" - LIGHTNING_DEBUG: "1" - TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" - # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 20 - - steps: - - uses: actions/checkout@v4 - - - name: basic setup - timeout-minutes: 20 - run: | - mkdir -p tests/_flagships - mkdir -p $VIDEO_LOCATION - pip --version - pip list - # for some reason the python package playwright is missing - pip install -r requirements/app/test.txt - python -m playwright install # --with-deps - - - name: Clone the Repo/App - uses: actions/checkout@v4 - with: - repository: ${{ matrix.repo }} - path: tests/_flagship-app - - - name: Adjust env. for this App - uses: ./.github/actions/prep-apps - with: - name: ${{ matrix.app }} - - - name: Install Lightning package - timeout-minutes: 20 - run: pip install -e .[cloud,test] -f $TORCH_URL - - name: List pip dependency - run: pip --version && pip list - - - name: Run the tests - working-directory: tests/ - env: - LIGHTNING_USER_ID: ${{ secrets[format('LIGHTNING_USER_ID_{0}', inputs.environment)] }} - LIGHTNING_API_KEY: ${{ secrets[format('LIGHTNING_API_KEY_{0}', inputs.environment)] }} - LIGHTNING_USERNAME: ${{ secrets[format('LIGHTNING_USERNAME_{0}', inputs.environment)] }} - LIGHTNING_CLOUD_URL: ${{ secrets[format('LIGHTNING_CLOUD_URL_{0}', inputs.environment)] }} - LAI_USER: ${{ secrets.LAI_SSH_USER }} - LAI_PASS: ${{ secrets.LAI_SSH_PASS }} - run: | - ls -l _flagship-app - python -m pytest integrations_app/flagship/test_${{ matrix.app }}.py \ - --capture=no -v --color=yes - - - name: Upload recordings - uses: actions/upload-artifact@v3 - if: failure() - with: - name: flahship-app-${{ matrix.app }} - path: ${{ env.VIDEO_LOCATION }} - - - name: Clean Previous Apps - if: always() - timeout-minutes: 3 - env: - LIGHTNING_USER_ID: ${{ secrets[format('LIGHTNING_USER_ID_{0}', inputs.environment)] }} - LIGHTNING_API_KEY: ${{ secrets[format('LIGHTNING_API_KEY_{0}', inputs.environment)] }} - LIGHTNING_USERNAME: ${{ secrets[format('LIGHTNING_USERNAME_{0}', inputs.environment)] }} - LIGHTNING_CLOUD_URL: ${{ secrets[format('LIGHTNING_CLOUD_URL_{0}', inputs.environment)] }} - LAI_USER: ${{ secrets.LAI_SSH_USER }} - LAI_PASS: ${{ secrets.LAI_SSH_PASS }} - run: | - time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" diff --git a/.github/workflows/ci-examples-app.yml b/.github/workflows/ci-examples-app.yml deleted file mode 100644 index b6db69e67aead..0000000000000 --- a/.github/workflows/ci-examples-app.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: Test App - examples - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - push: - branches: [master, "release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - ".actions/*" - - ".github/workflows/ci-examples-app.yml" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "tests/integrations_app/**" - - "!tests/integrations_app/flagship/**" - - "examples/app/**" - - "requirements/app/**" - - "setup.py" - - "!requirements/*/docs.txt" - - "!*.md" - - "!**/*.md" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -defaults: - run: - shell: bash - -jobs: - app-examples: - if: github.event.pull_request.draft == false - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-20.04, macOS-12, windows-2022] - pkg-name: ["lightning"] - python-version: ["3.9"] - requires: ["oldest", "latest"] - include: - # "app" installs the standalone package - - { os: "macOS-12", pkg-name: "app", python-version: "3.9", requires: "latest" } - - { os: "ubuntu-20.04", pkg-name: "app", python-version: "3.9", requires: "latest" } - - { os: "windows-2022", pkg-name: "app", python-version: "3.9", requires: "latest" } - # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 15 - env: - PACKAGE_NAME: ${{ matrix.pkg-name }} - FREEZE_REQUIREMENTS: 1 - TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" - PYPI_CACHE_DIR: "_pip-wheels" - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: basic setup - run: pip install -q -r .actions/requirements.txt - - - name: Set min. dependencies - if: ${{ matrix.requires == 'oldest' }} - run: python .actions/assistant.py replace_oldest_ver - - - name: pip wheels cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.PYPI_CACHE_DIR }} - key: pypi_wheels - - - name: Install Lightning package & dependencies - timeout-minutes: 20 - run: | - extra=$(python -c "print({'lightning': 'app-'}.get('${{ matrix.pkg-name }}', ''))") - # do not use `-e` because it will make both packages available since it adds `src` to `sys.path` automatically - pip install ".[${extra}dev]" -U -f ${TORCH_URL} -f ${PYPI_CACHE_DIR} --prefer-binary - pip list - - name: Dump handy wheels - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - continue-on-error: true - uses: ./.github/actions/pip-wheels - with: - wheel-dir: ${{ env.PYPI_CACHE_DIR }} - torch-url: ${{ env.TORCH_URL }} - cache-key: "pypi_wheels" - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "16" - - - name: Install Yarn - timeout-minutes: 20 - run: npm install -g yarn - - - name: Adjust imports -> App - if: ${{ matrix.pkg-name != 'lightning' }} - run: | - python .actions/assistant.py copy_replace_imports --source_dir="./tests" \ - --source_import="lightning.app,lightning.fabric,lightning.pytorch" \ - --target_import="lightning_app,lightning_fabric,pytorch_lightning" \ - --lightning_by="lightning_app" - python .actions/assistant.py copy_replace_imports --source_dir="./examples" \ - --source_import="lightning.app,lightning.fabric,lightning.pytorch,lightning" \ - --target_import="lightning_app,lightning_fabric,pytorch_lightning,lightning_app" - - - name: Switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV - - - name: Tests - working-directory: ./tests - run: | - python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \ - -m pytest -m "not cloud" integrations_app \ - --timeout=120 --durations=0 -vvvv - - - name: Statistics - if: success() - working-directory: ./tests - run: | - coverage xml -i - coverage report -i - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: tests/coverage.xml - flags: cpu,pytest,app,examples - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: false diff --git a/.github/workflows/ci-flagship-apps.yml b/.github/workflows/ci-flagship-apps.yml deleted file mode 100644 index 6332299fc03d1..0000000000000 --- a/.github/workflows/ci-flagship-apps.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Test App - flagships - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - push: - branches: ["release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] - paths: - - ".github/workflows/_flagship-apps.yml" - - ".github/workflows/ci-flagship-apps.yml" - - "github/actions/prep-apps/action.yml" - - "tests/integrations_app/flagship/**" - schedule: - # on Sundays - - cron: "0 0 * * 0" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - test-flagships: - if: github.event.pull_request.draft == false - uses: ./.github/workflows/_flagship-apps.yml - secrets: inherit diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index 67a9b9f21b515..6e82167410ec3 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -43,14 +43,8 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04", "macOS-12", "windows-2022"] - pkg-name: ["app", "fabric", "pytorch", "lightning", "notset"] + pkg-name: ["fabric", "pytorch", "lightning", "notset"] python-version: ["3.8", "3.11"] - # TODO: add also install from source - include: - - { os: "macOS-12", pkg-name: "lightning", python-version: "3.9", pkg-extra: "app" } - - { os: "macOS-12", pkg-name: "notset", python-version: "3.9", pkg-extra: "app" } - - { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.9", pkg-extra: "app" } - - { os: "ubuntu-22.04", pkg-name: "notset", python-version: "3.9", pkg-extra: "app" } steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -70,16 +64,6 @@ jobs: with: pkg-folder: dist/${{ env.PKG_DIR }} pkg-name: ${{ matrix.pkg-name }} - pkg-extra: ${{ matrix.pkg-extra }} - - - name: Run CLI (via python) - if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app' }} - run: python -m lightning --version - - name: Run CLI (direct bash) - if: | - ((matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app') || - matrix.pkg-name == 'app' - run: lightning_app --version - name: DocTests actions working-directory: .actions/ @@ -88,15 +72,15 @@ jobs: python -m pytest assistant.py - name: Adjust code for standalone - if: contains(fromJSON('["app", "fabric", "pytorch"]'), matrix.pkg-name) + if: contains(fromJSON('["fabric", "pytorch"]'), matrix.pkg-name) run: | python .actions/assistant.py copy_replace_imports --source_dir="./src" \ - --source_import="lightning.pytorch,lightning.fabric,lightning.app" \ - --target_import="pytorch_lightning,lightning_fabric,lightning_app" + --source_import="lightning.pytorch,lightning.fabric" \ + --target_import="pytorch_lightning,lightning_fabric" - name: Rename src folders working-directory: src/ run: | - python -c "n = '${{matrix.pkg-name}}' ; n = n if n in ('app', 'fabric', 'pytorch') else '' ; print('PKG_NAME=' + n)" >> $GITHUB_ENV + python -c "n = '${{matrix.pkg-name}}' ; n = n if n in ('fabric', 'pytorch') else '' ; print('PKG_NAME=' + n)" >> $GITHUB_ENV rm -f ./*/__*.py rm -f ./**/__*.py mv lightning lit # rename lightning folder to prevent accidental local imports @@ -104,12 +88,8 @@ jobs: if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }} working-directory: src/lit run: | - items=("data" "store" "app") + items=("data") for item in "${items[@]}"; do - if [[ "$item" == "${{ matrix.pkg-extra }}" ]]; then - echo "Skipping $item" - continue # Skip this iteration - fi echo "Removing $item" rm -rf $item done diff --git a/.github/workflows/ci-tests-app.yml b/.github/workflows/ci-tests-app.yml deleted file mode 100644 index ee643fa397f43..0000000000000 --- a/.github/workflows/ci-tests-app.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Test App - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - push: - branches: [master, "release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - ".actions/*" - - ".github/workflows/ci-tests-app.yml" - - "requirements/ci.txt" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "tests/tests_app/**" - - "requirements/app/**" - - "setup.py" - - "!requirements/*/docs.txt" - - "!*.md" - - "!**/*.md" - schedule: - # At the end of every day - - cron: "0 0 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -defaults: - run: - shell: bash - -jobs: - app-pytest: - if: github.event.pull_request.draft == false - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-20.04", "macOS-12", "windows-2022"] - pkg-name: ["lightning"] - python-version: ["3.8"] - requires: ["oldest", "latest"] - include: - # only run Python latest, use App scope to limit dependency issues - - { os: "macOS-12", pkg-name: "app", python-version: "3.11", requires: "latest" } - - { os: "ubuntu-22.04", pkg-name: "app", python-version: "3.11", requires: "latest" } - - { os: "windows-2022", pkg-name: "app", python-version: "3.11", requires: "latest" } - # "app" installs the standalone package - - { os: "macOS-12", pkg-name: "app", python-version: "3.9", requires: "latest" } - - { os: "ubuntu-20.04", pkg-name: "app", python-version: "3.9", requires: "latest" } - - { os: "windows-2022", pkg-name: "app", python-version: "3.8", requires: "latest" } - # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 55 - env: - PACKAGE_NAME: ${{ matrix.pkg-name }} - FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} - PYPI_CACHE_DIR: "_pip-wheels" - TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: basic setup - run: pip install -q -r .actions/requirements.txt - - - name: Set min. dependencies - if: ${{ matrix.requires == 'oldest' }} - run: python .actions/assistant.py replace_oldest_ver - - - name: pip wheels cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.PYPI_CACHE_DIR }} - key: pypi_wheels - - name: List restored pkgs - run: | - mkdir -p $PYPI_CACHE_DIR - ls -lh $PYPI_CACHE_DIR - - - name: Env. variables - run: | - # Switch coverage scope - python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV - # if you install mono-package set dependency only for this subpackage - python -c "print('EXTRA_PREFIX=' + str('' if '${{matrix.pkg-name}}' != 'lightning' else 'app-'))" >> $GITHUB_ENV - - - name: Install package & dependencies - timeout-minutes: 20 - run: | - pip install -e ".[${EXTRA_PREFIX}dev]" -U --prefer-binary \ - --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" - pip list - - name: Dump handy wheels - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - continue-on-error: true - uses: ./.github/actions/pip-wheels - with: - wheel-dir: ${{ env.PYPI_CACHE_DIR }} - torch-url: ${{ env.TORCH_URL }} - cache-key: "pypi_wheels" - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "16" - - name: Install Yarn - timeout-minutes: 20 - run: npm install -g yarn - - - name: Adjust imports -> App - if: ${{ matrix.pkg-name != 'lightning' }} - run: | - python .actions/assistant.py copy_replace_imports --source_dir="./tests" \ - --source_import="lightning.app,lightning.fabric,lightning.pytorch" \ - --target_import="lightning_app,lightning_fabric,pytorch_lightning" \ - --lightning_by="lightning_app" - python .actions/assistant.py copy_replace_imports --source_dir="./examples" \ - --source_import="lightning.app,lightning.fabric,lightning.pytorch" \ - --target_import="lightning_app,lightning_fabric,pytorch_lightning" \ - --lightning_by="lightning_app" - - - name: Switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV - - - name: Set parallel for Unix - if: ${{ runner.os != 'windows' }} - # on Win, tests takes even loner then with normal single thread - run: echo "PYTEST_XDIST_ARGS=-n auto --dist=loadfile" >> $GITHUB_ENV - - name: Tests - working-directory: ./tests - run: | - set -e - python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \ - -m pytest -m "not cloud" -vvvv tests_app \ - --ignore="tests_app/components/python/test_python.py" \ - --timeout=120 --durations=50 ${PYTEST_XDIST_ARGS} - pytest -m "not cloud" -v \ - tests_app/components/python/test_python.py \ - --timeout=120 --durations=50 - - - name: Statistics - if: success() - working-directory: ./tests - run: | - coverage xml -i - coverage report -i - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: tests/coverage.xml - flags: ${{ env.COVERAGE_SCOPE }},cpu,pytest - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: false -# TODO: figure out why we clone and install quick-start -# - name: Clone Quick Start Example Repo -# uses: actions/checkout@v4 -# # TODO: this needs to be git submodule -# if: matrix.os == 'windows-2022' # because the install doesn't work on windows -# with: -# repository: Lightning-AI/lightning-quick-start -# ref: 'main' -# path: lightning-quick-start -# -# - name: Lightning Install quick-start -# if: matrix.os != 'windows-2022' # because the install doesn't work on windows -# run: | -# python -m lightning install app lightning/quick-start -y diff --git a/.github/workflows/ci-tests-store.yml b/.github/workflows/ci-tests-store.yml deleted file mode 100644 index 60614005c614d..0000000000000 --- a/.github/workflows/ci-tests-store.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Test Store - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - push: - branches: [master, "release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - ".actions/*" - - "requirements/ci.txt" - - "requirements/store/**" - - "src/lightning/__init__.py" - - "src/lightning/__setup__.py" - - "src/lightning/__version__.py" - - "src/lightning/store/**" - - "tests/tests_store/**" - - "pyproject.toml" # includes pytest config - - ".github/workflows/ci-tests-store.yml" - - "!requirements/*/docs.txt" - - "!*.md" - - "!**/*.md" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -defaults: - run: - shell: bash - -jobs: - store-cpu: - runs-on: ${{ matrix.os }} - if: github.event.pull_request.draft == false - strategy: - fail-fast: false - matrix: - os: ["macOS-11", "ubuntu-20.04", "windows-2022"] - pkg-name: ["lightning"] - python-version: ["3.10"] - pytorch-version: ["2.0"] - timeout-minutes: 25 # because of building grpcio on Mac - env: - PACKAGE_NAME: ${{ matrix.pkg-name }} - FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} - # PYPI_CACHE_DIR: "_pip-wheels" - TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/torch_stable.html" - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Adjust PyTorch versions in requirements files - if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }} - run: | - pip install -q -r requirements/ci.txt - python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py - for fpath in `ls requirements/store/*.txt`; do \ - python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ - done - - - name: Install package & dependencies - timeout-minutes: 20 - run: | - pip install -e ".[store,store-test]" -U -f ${TORCH_URL} --prefer-binary - pip list - - - name: Testing Store - working-directory: tests/tests_store - # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 - run: | - python -m coverage run --source lightning \ - -m pytest -v --timeout=60 --durations=60 - - - name: Statistics - if: success() - working-directory: tests/tests_store - run: | - coverage report - coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - # see: https://github.com/actions/toolkit/issues/399 - continue-on-error: true - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: tests/tests_store/coverage.xml - flags: lightning,cpu,pytest,python${{ matrix.python-version }} - name: CPU-coverage - fail_ci_if_error: false diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 535274e403a58..55e2a8ec4a01a 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -44,7 +44,6 @@ jobs: FREEZE_REQUIREMENTS: 1 timeout-minutes: 20 run: | - # TODO: investigate hanging installation with app sub-package pip install -e '.[pytorch-all,fabric-all]' -r requirements/typing.txt pip list diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index 7f7a09ce44007..4a9889eae02e6 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -15,8 +15,6 @@ on: - "docs/**" - "_notebooks" - "requirements/**" - - "src/lightning/app/**" - - "src/lightning_app/*" - "src/lightning/fabric/**" - "src/lightning_fabric/*" - "src/lightning/pytorch/**" @@ -58,7 +56,7 @@ jobs: strategy: fail-fast: false matrix: - pkg-name: ["app", "fabric", "pytorch"] + pkg-name: ["fabric", "pytorch"] target: ["html", "doctest", "linkcheck"] env: DOCS_COPY_NOTEBOOKS: 1 @@ -144,7 +142,7 @@ jobs: strategy: fail-fast: false matrix: - pkg-name: ["app", "fabric", "pytorch"] + pkg-name: ["fabric", "pytorch"] env: GCP_TARGET: "gs://lightning-docs-${{ matrix.pkg-name }}" # use input if dispatch or git tag diff --git a/.github/workflows/release-pkg.yml b/.github/workflows/release-pkg.yml index 696efd8b68291..e922a34446d97 100644 --- a/.github/workflows/release-pkg.yml +++ b/.github/workflows/release-pkg.yml @@ -165,7 +165,7 @@ jobs: strategy: fail-fast: false matrix: - name: ["APP", "FABRIC", "PYTORCH", "LIGHTNING"] + name: ["FABRIC", "PYTORCH", "LIGHTNING"] steps: - uses: actions/checkout@v4 # needed for local action below - uses: actions/download-artifact@v3 @@ -190,7 +190,7 @@ jobs: strategy: fail-fast: false matrix: - name: ["APP", "FABRIC", "PYTORCH", "LIGHTNING"] + name: ["FABRIC", "PYTORCH", "LIGHTNING"] steps: - uses: actions/checkout@v4 # needed for local action below - uses: actions/download-artifact@v3 diff --git a/.gitignore b/.gitignore index de1de44fec235..cf5face7db3d4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,6 @@ lightning_logs/ # Documentations docs/venv*/ docs/build*/ -docs/source-app/*/api -docs/source-app/generated -docs/source-app/*/generated -docs/source-app/_static/fetched-s3-assets docs/source-fabric/_static/fetched-s3-assets docs/source-pytorch/api docs/source-pytorch/*.md @@ -59,7 +55,6 @@ wheels/ .installed.cfg *.egg src/*/version.info -src/lightning_app/* src/lightning_fabric/* src/pytorch_lightning/* !src/*/__about__.py @@ -182,8 +177,6 @@ cifar-10-batches-py # ctags tags .tags -src/lightning_app/ui/* -src/lightning/app/ui/* *examples/template_react_ui* hars* artifacts/* diff --git a/.lightningignore b/.lightningignore deleted file mode 100644 index 4ce8d526e30e3..0000000000000 --- a/.lightningignore +++ /dev/null @@ -1,16 +0,0 @@ -_notebooks -.azure -.github -.ipynb_checkpoints -.pytest_cache -.shared -.storage -.venv -.vscode -.git -artifacts -Datasets -dist -docs -examples -tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 834a903bcac19..fbf4b2de6a999 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,7 +90,6 @@ repos: - mdformat_frontmatter exclude: | (?x)^( - src/lightning/app/CHANGELOG.md| src/lightning/fabric/CHANGELOG.md| src/lightning/pytorch/CHANGELOG.md| README.md diff --git a/.readthedocs.yml b/.readthedocs.yml index a42fbb7e88214..625c56a5fe61b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -42,8 +42,7 @@ build: - pip install -U pip awscli py-tree --user - python -m awscli s3 sync --no-sign-request s3://sphinx-packages/ dist/ ; ls -lh dist/ - > - pip install -e ".[app]" -q -r _notebooks/.actions/requires.txt \ - -r requirements/app/docs.txt \ + pip install -e . -q -r _notebooks/.actions/requires.txt \ -r requirements/fabric/docs.txt \ -r requirements/pytorch/docs.txt \ -f 'https://download.pytorch.org/whl/cpu/torch_stable.html' -f dist/ ; diff --git a/Makefile b/Makefile index 2f7ca37222a10..426c18042994c 100644 --- a/Makefile +++ b/Makefile @@ -21,13 +21,10 @@ clean: rm -rf ./docs/source-pytorch/generated rm -rf ./docs/source-pytorch/*/generated rm -rf ./docs/source-pytorch/api - rm -rf ./docs/source-app/generated - rm -rf ./docs/source-app/*/generated rm -rf build rm -rf dist rm -rf *.egg-info rm -rf src/*.egg-info - rm -rf src/lightning_app/*/ rm -rf src/lightning_fabric/*/ rm -rf src/pytorch_lightning/*/ @@ -35,14 +32,11 @@ test: clean # Review the CONTRIBUTING documentation for other ways to test. pip install -e . \ -r requirements/pytorch/base.txt \ - -r requirements/app/app.txt \ -r requirements/fabric/base.txt \ -r requirements/pytorch/test.txt \ - -r requirements/app/test.txt # run tests with coverage python -m coverage run --source src/lightning/pytorch -m pytest src/lightning/pytorch tests/tests_pytorch -v - python -m coverage run --source src/lightning/app -m pytest tests/tests/app -v python -m coverage run --source src/lightning/fabric -m pytest src/lightning/fabric tests/tests_fabric -v python -m coverage report @@ -54,10 +48,6 @@ sphinx-theme: aws s3 sync --no-sign-request s3://sphinx-packages/ dist/ pip install lai-sphinx-theme -f dist/ -docs-app: clean sphinx-theme - pip install -e .[all] --quiet -r requirements/app/docs.txt - cd docs/source-app && $(MAKE) html --jobs $(nproc) - docs-fabric: clean sphinx-theme pip install -e .[all] --quiet -r requirements/fabric/docs.txt cd docs/source-fabric && $(MAKE) html --jobs $(nproc) diff --git a/docs/crossroad.html b/docs/crossroad.html index fc072aba60df5..0c25930c6343a 100644 --- a/docs/crossroad.html +++ b/docs/crossroad.html @@ -9,7 +9,6 @@ - diff --git a/docs/rtfd-build.sh b/docs/rtfd-build.sh index 2aa6928f10a19..1b1d5dcab319b 100644 --- a/docs/rtfd-build.sh +++ b/docs/rtfd-build.sh @@ -5,7 +5,7 @@ if ! [ $READTHEDOCS_VERSION == "latest" -o $READTHEDOCS_VERSION == "stable" ]; then export FAST_DOCS_DEV=1 ; root=$(pwd) ; - for pkg in 'app' 'fabric' 'pytorch' ; + for pkg in 'fabric' 'pytorch' ; do cd $root/docs/source-$pkg ; make html --jobs $(nproc) ; diff --git a/docs/source-app/Makefile b/docs/source-app/Makefile deleted file mode 100644 index 268e09561bb72..0000000000000 --- a/docs/source-app/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -T -W -SPHINXBUILD = sphinx-build -SOURCEDIR = . -BUILDDIR = ../build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source-app/_static/copybutton.js b/docs/source-app/_static/copybutton.js deleted file mode 100644 index aef241ab6ac40..0000000000000 --- a/docs/source-app/_static/copybutton.js +++ /dev/null @@ -1,78 +0,0 @@ -/* Copied from the official Python docs: https://docs.python.org/3/_static/copybutton.js */ -$(document).ready(function () { - /* Add a [>>>] button on the top-right corner of code samples to hide - * the >>> and ... prompts and the output and thus make the code - * copyable. */ - var div = $( - ".highlight-python .highlight," + - ".highlight-python3 .highlight," + - ".highlight-pycon .highlight," + - ".highlight-default .highlight", - ); - var pre = div.find("pre"); - - // get the styles from the current theme - pre.parent().parent().css("position", "relative"); - var hide_text = "Hide the prompts and output"; - var show_text = "Show the prompts and output"; - var border_width = pre.css("border-top-width"); - var border_style = pre.css("border-top-style"); - var border_color = pre.css("border-top-color"); - var button_styles = { - cursor: "pointer", - position: "absolute", - top: "0", - right: "0", - "border-color": border_color, - "border-style": border_style, - "border-width": border_width, - color: border_color, - "text-size": "75%", - "font-family": "monospace", - "padding-left": "0.2em", - "padding-right": "0.2em", - "border-radius": "0 3px 0 0", - }; - - // create and add the button to all the code blocks that contain >>> - div.each(function (index) { - var jthis = $(this); - if (jthis.find(".gp").length > 0) { - var button = $('>>>'); - button.css(button_styles); - button.attr("title", hide_text); - button.data("hidden", "false"); - jthis.prepend(button); - } - // tracebacks (.gt) contain bare text elements that need to be - // wrapped in a span to work with .nextUntil() (see later) - jthis - .find("pre:has(.gt)") - .contents() - .filter(function () { - return this.nodeType == 3 && this.data.trim().length > 0; - }) - .wrap(""); - }); - - // define the behavior of the button when it's clicked - $(".copybutton").click(function (e) { - e.preventDefault(); - var button = $(this); - if (button.data("hidden") === "false") { - // hide the code output - button.parent().find(".go, .gp, .gt").hide(); - button.next("pre").find(".gt").nextUntil(".gp, .go").css("visibility", "hidden"); - button.css("text-decoration", "line-through"); - button.attr("title", show_text); - button.data("hidden", "true"); - } else { - // show the code output - button.parent().find(".go, .gp, .gt").show(); - button.next("pre").find(".gt").nextUntil(".gp, .go").css("visibility", "visible"); - button.css("text-decoration", "none"); - button.attr("title", hide_text); - button.data("hidden", "false"); - } - }); -}); diff --git a/docs/source-app/_static/images/icon.svg b/docs/source-app/_static/images/icon.svg deleted file mode 100644 index e88fc19036178..0000000000000 --- a/docs/source-app/_static/images/icon.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/docs/source-app/_static/images/logo-large.svg b/docs/source-app/_static/images/logo-large.svg deleted file mode 100644 index 39531f95e9dba..0000000000000 --- a/docs/source-app/_static/images/logo-large.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/docs/source-app/_static/images/logo-small.svg b/docs/source-app/_static/images/logo-small.svg deleted file mode 100644 index 1f523a57c4a16..0000000000000 --- a/docs/source-app/_static/images/logo-small.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/docs/source-app/_static/images/logo.png b/docs/source-app/_static/images/logo.png deleted file mode 100644 index 308a6ee419d44..0000000000000 Binary files a/docs/source-app/_static/images/logo.png and /dev/null differ diff --git a/docs/source-app/_static/images/logo.svg b/docs/source-app/_static/images/logo.svg deleted file mode 100644 index b73eaa8cedb50..0000000000000 --- a/docs/source-app/_static/images/logo.svg +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source-app/_static/main.css b/docs/source-app/_static/main.css deleted file mode 100644 index c1bd8ad0305b7..0000000000000 --- a/docs/source-app/_static/main.css +++ /dev/null @@ -1,3 +0,0 @@ -col { - width: 50% !important; -} diff --git a/docs/source-app/_templates/classtemplate.rst b/docs/source-app/_templates/classtemplate.rst deleted file mode 100644 index 5b7f465516787..0000000000000 --- a/docs/source-app/_templates/classtemplate.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. role:: hidden - :class: hidden-section -.. currentmodule:: {{ module }} - - -{{ name | underline }} - -.. autoclass:: {{ name }} - :members: diff --git a/docs/source-app/_templates/classtemplate_no_index.rst b/docs/source-app/_templates/classtemplate_no_index.rst deleted file mode 100644 index 858c37b51567a..0000000000000 --- a/docs/source-app/_templates/classtemplate_no_index.rst +++ /dev/null @@ -1,12 +0,0 @@ -:orphan: - -.. role:: hidden - :class: hidden-section -.. currentmodule:: {{ module }} - - -{{ name | underline }} - -.. autoclass:: {{ name }} - :members: - :noindex: diff --git a/docs/source-app/_templates/layout.html b/docs/source-app/_templates/layout.html deleted file mode 100644 index 90e6125f9ad28..0000000000000 --- a/docs/source-app/_templates/layout.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "!layout.html" %} - - -{% block footer %} {{ super() }} - - -{% endblock %} diff --git a/docs/source-app/_templates/theme_variables.jinja b/docs/source-app/_templates/theme_variables.jinja deleted file mode 100644 index 914f8dcafc96b..0000000000000 --- a/docs/source-app/_templates/theme_variables.jinja +++ /dev/null @@ -1,18 +0,0 @@ -{%- set external_urls = { - 'github': 'https://github.com/Lightning-AI/lightning', - 'github_issues': 'https://github.com/Lightning-AI/lightning/issues', - 'contributing': 'https://github.com/Lightning-AI/lightning/blob/master/.github/CONTRIBUTING.md', - 'governance': 'https://github.com/Lightning-AI/lightning/blob/master/docs/source-pytorch/governance.rst', - 'docs': 'https://lightning.rtfd.io/en/latest', - 'twitter': 'https://twitter.com/PyTorchLightnin', - 'discuss': 'https://discord.gg/VptPCZkGNa', - 'tutorials': 'https://pt-lightning.readthedocs.io/en/latest/#tutorials', - 'previous_pytorch_versions': 'https://pt-lightning.rtfd.io/en/latest/', - 'home': 'https://lightning.ai/', - 'get_started': 'https://pt-lightning.readthedocs.io/en/latest/introduction_guide.html', - 'features': 'https://pt-lightning.rtfd.io/en/latest/', - 'blog': 'https://www.pytorchlightning.ai/blog', - 'resources': 'https://pt-lightning.readthedocs.io/en/latest/#community-examples', - 'support': 'https://pt-lightning.rtfd.io/en/latest/', -} --%} diff --git a/docs/source-app/api_reference/components.rst b/docs/source-app/api_reference/components.rst deleted file mode 100644 index 69d53b79e76ce..0000000000000 --- a/docs/source-app/api_reference/components.rst +++ /dev/null @@ -1,35 +0,0 @@ -######################## -lightning.app.components -######################## - -.. contents:: - :depth: 1 - :local: - :backlinks: top - -.. currentmodule:: lightning.app.components - - -Built-in Components -___________________ - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate.rst - - ~database.client.DatabaseClient - ~database.server.Database - ~python.popen.PopenPythonScript - ~python.tracer.TracerPythonScript - ~training.LightningTrainerScript - ~serve.gradio_server.ServeGradio - ~serve.serve.ModelInferenceAPI - ~serve.python_server.PythonServer - ~serve.streamlit.ServeStreamlit - ~multi_node.base.MultiNode - ~multi_node.fabric.FabricMultiNode - ~multi_node.pytorch_spawn.PyTorchSpawnMultiNode - ~multi_node.trainer.LightningTrainerMultiNode - ~serve.auto_scaler.AutoScaler - ~serve.auto_scaler.ColdStartProxy diff --git a/docs/source-app/api_reference/core.rst b/docs/source-app/api_reference/core.rst deleted file mode 100644 index 324f3c448978f..0000000000000 --- a/docs/source-app/api_reference/core.rst +++ /dev/null @@ -1,26 +0,0 @@ -:orphan: - -################## -lightning.app.core -################## - -.. contents:: - :depth: 1 - :local: - :backlinks: top - -.. currentmodule:: lightning.app.core - -Core APIs -___________________ - -.. autosummary:: - :toctree: api/ - :nosignatures: - :template: classtemplate.rst - - LightningApp - LightningFlow - LightningWork - -Learn more about :ref:`Lightning Core `. diff --git a/docs/source-app/api_reference/frontend.rst b/docs/source-app/api_reference/frontend.rst deleted file mode 100644 index 514b2cf35bc75..0000000000000 --- a/docs/source-app/api_reference/frontend.rst +++ /dev/null @@ -1,25 +0,0 @@ -###################### -lightning.app.frontend -###################### - -.. contents:: - :depth: 1 - :local: - :backlinks: top - -.. currentmodule:: lightning.app.frontend - -Lightning FrontEnds -___________________ - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate.rst - - ~frontend.Frontend - ~web.StaticWebFrontend - ~stream_lit.StreamlitFrontend - ~panel.PanelFrontend - -Learn more about :ref:`Frontend's `. diff --git a/docs/source-app/api_reference/runners.rst b/docs/source-app/api_reference/runners.rst deleted file mode 100644 index f7e550b7c7733..0000000000000 --- a/docs/source-app/api_reference/runners.rst +++ /dev/null @@ -1,21 +0,0 @@ -##################### -lightning.app.runners -##################### - -.. contents:: - :depth: 1 - :local: - :backlinks: top - -.. currentmodule:: lightning.app.runners - -Lightning Core -______________ - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate.rst - - ~cloud.CloudRuntime - ~multiprocess.MultiProcessRuntime diff --git a/docs/source-app/api_reference/storage.rst b/docs/source-app/api_reference/storage.rst deleted file mode 100644 index 3173914427586..0000000000000 --- a/docs/source-app/api_reference/storage.rst +++ /dev/null @@ -1,71 +0,0 @@ -##################### -lightning.app.storage -##################### - -Lightning Core -______________ - -.. contents:: - :depth: 1 - :local: - :backlinks: top - -.. currentmodule:: lightning.app.storage - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate.rst - - ~path.Path - ~drive.Drive - ~payload.Payload - ~mount.Mount - ----- - -************************ -Learn more about Storage -************************ - -.. raw:: html - -
-
- -.. displayitem:: - :header: Learn about the differences between Drive vs Path. - :description: Learn about their differences. - :col_css: col-md-4 - :button_link: ../glossary/storage/differences.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: The Drive Object. - :description: Put, List and Get Files From a Shared Drive Disk. - :col_css: col-md-4 - :button_link: ../glossary/storage/drive.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: The Path Object. - :description: Transfer Files From One Component to Another by Reference. - :col_css: col-md-4 - :button_link: ../glossary/storage/path.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: The Mount Object. - :description: Mount an AWS S3 Bucket When Running on the Cloud. - :col_css: col-md-4 - :button_link: ../workflows/mount_aws_s3_bucket.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/basics.rst b/docs/source-app/basics.rst deleted file mode 100644 index 69b1bf751014b..0000000000000 --- a/docs/source-app/basics.rst +++ /dev/null @@ -1,259 +0,0 @@ -:orphan: - -.. _basics: - -###### -Basics -###### - -In this guide, we'll cover the basic terminology associated with the Lightning framework. - ----- - -************** -Lightning App -************** - -The :class:`~lightning.app.core.app.LightningApp` runs a tree of one or more components that interact to create end-to-end applications. There are two kinds of components: :class:`~lightning.app.core.flow.LightningFlow` and :class:`~lightning.app.core.work.LightningWork`. This modular design enables you to reuse components created by other users. - ----- - -Lightning Work -^^^^^^^^^^^^^^ - -The :class:`~lightning.app.core.work.LightningWork` component is a building block optimized for long-running jobs or integrating third-party services. LightningWork can be used for training large models, downloading a dataset, or any long-lasting operation. - ----- - -Lightning Flow -^^^^^^^^^^^^^^ - -The :class:`~lightning.app.core.flow.LightningFlow` component coordinates long-running tasks :class:`~lightning.app.core.work.LightningWork` and runs its children :class:`~lightning.app.core.flow.LightningFlow` components. - ----- - -Lightning App Tree -^^^^^^^^^^^^^^^^^^ - -Components can be nested to form component trees where the LightningFlows are its branches and LightningWorks are its leaves. - -Here's a basic application with four flows and two works: - -.. literalinclude:: code_samples/quickstart/app_comp.py - -And here's its associated tree structure: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/tree.gif - :alt: Basic App Components - :width: 100 % - -A Lightning App runs all flows into a single process. Its flows coordinate the execution of the works each running in their own independent processes. - ----- - -Lightning Distributed Event Loop -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Drawing inspiration from modern web frameworks like `React.js `_, the Lightning app runs all flows in an **event loop** (forever), which is triggered every 0.1 seconds after collecting any works' state change. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_loop.gif - -When running an app in the cloud, the :class:`~lightning.app.core.work.LightningWork` run on different machines. Lightning communicates any :class:`~lightning.app.core.work.LightningWork` state changes to the **event loop** which re-executes the flow with the newly-collected works' state. - ----- - -Lightning App State -^^^^^^^^^^^^^^^^^^^ - -By design, each component is stateful and its state is composed of all its attributes. The **Lightning App State** is the collection of all its components state. - -With this mechanism, any component can **react** to any other component **state changes**, simply by relying on its attributes within the flow. - -For example, here we define two flow components, **RootFlow** and **ChildFlow**, where the child flow prints and increments a counter indefinitely and gets reflected in **RootFlow** state. - -You can easily check the state of your entire app: - -.. literalinclude:: code_samples/quickstart/app_01.py - -Here's the entire tree structure associated with your app: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/parent_child.png - :alt: Parent Child Components - :width: 100 % - -And here's the output you get when running the above application using **Lightning CLI**: - -.. code-block:: console - - $ lightning_app run app docs/source/code_samples/quickstart/app_01.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - State: {'works': {'w_1': {'vars': {'counter': 1}}, 'w_2': {'vars': {'counter': 0}}}} - - State: {'works': {'w_1': {'vars': {'counter': 3}}, 'w_2': {'vars': {'counter': 1}}}} - - State: {'works': {'w_1': {'vars': {'counter': 4}}, 'w_2': {'vars': {'counter': 1}}}} - - State: {'works': {'w_1': {'vars': {'counter': 5}}, 'w_2': {'vars': {'counter': 2}}}} - - State: {'works': {'w_1': {'vars': {'counter': 6}}, 'w_2': {'vars': {'counter': 2}}}} - - State: {'works': {'w_1': {'vars': {'counter': 7}}, 'w_2': {'vars': {'counter': 3}}}} - ... - -This app will count forever because the **lightning event loop** indefinitely calls the root flow run method. - ----- - -******************************* -Controlling the Execution Flow -******************************* - - -LightningWork: To Cache or Not to Cache Calls -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -With Lightning, you can control how to run your components. - -By default, the :class:`~lightning.app.core.flow.LightningFlow` is executed infinitely by the **Lightning Infinite Loop** and the :class:`~lightning.app.core.work.LightningWork` does not run in **parallel**, -meaning the **Lightning Infinite Loop** (a.k.a the flow) waits until that long-running work is completed to continue. - -Similar to `React.js Components and Props `_, the :class:`~lightning.app.core.work.LightningWork` -component accepts arbitrary inputs (the "props") to its **run** method and by default runs **once** for each unique input provided. - -Here's an example of this behavior: - -.. literalinclude:: code_samples/basics/0.py - :language: python - :emphasize-lines: 10, 19 - -And you should see the following by running the code above: - -.. code-block:: console - - $ python example.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 10} - -As you can see, the intermediate run didn't execute as already cached. - -To disable this behavior, set ``cache_calls=False`` to make any LightningWork run infinitely. - -.. literalinclude:: code_samples/basics/1.py - :diff: code_samples/basics/0.py - -.. code-block:: console - - $ python example.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 10} - - -.. note:: Passing a sequence of different props to the work run method queues their execution. We recommend avoiding this behavior as it can be hard to debug. Instead, wait for the previous run to execute. - ----- - -LightningWork: Parallel vs Non Parallel -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The LightningWork component is made for long-running jobs. - -As an example, let's create a long-running **LightningWork** component that will take 1 hour to do its "work". - -.. literalinclude:: code_samples/quickstart/app_02.py - :language: python - :emphasize-lines: 15 - -Here's the output you get when running the above application using **Lightning CLI**: - -.. code-block:: console - - $ lightning_app run app docs/source/code_samples/quickstart/app_02.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - 0.0 0.0 - ... - 0.0003 0.0003 - ... - 1.0 1.0 - ... - 1 hour later! - 1.0 1.0 - 1 hour later! - 1.0 1.0 - 1 hour later! - ... - -The child work runs only once, hence why the progress counter stops increasing once the work is completed. - -This is useful for monitoring the progress of a long-running operation, like training a big model. - -.. note :: - The Lightning Infinite Loop runs multiple cycles per second. - It is good practice to keep the loop running fast, so that your application stays responsive, - especially when it contains user-interface components. - ----- - -**************** -Multiple works -**************** - -In practical use cases, you might want to execute multiple long-running works in parallel. - -To enable this behavior, set ``parallel=True`` in the ``__init__`` method of -your :class:`~lightning.app.core.work.LightningWork`. - -Here's an example of the interaction between parallel and non-parallel behaviors: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/blocking_non_blocking.gif - :alt: mnist GPU bar - :width: 100 % - -Below, we reuse the **HourLongWork** work defined in the previous example, but modify the **RootFlow** -to run two **HourLongWork** works in a parallel way. - -.. literalinclude:: code_samples/quickstart/app/app_0.py - :emphasize-lines: 21 - -Above, both ``child_work_1`` and ``child_work_2`` are long-running works that are executed -asynchronously in parallel. - -When running the above app, we see the following logs: - -.. code-block:: console - - $ lightning_app run app docs/source/code_samples/quickstart/app/app_0.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - 0.0, 0.0 - ... - 0.0003, 0.0003 - ... - 1.0, 1.0 - ... - 1 hour later `child_work_1` started! - 1 hour later `child_work_2` started! - 0.0, 0.0 - ... - 0.0003, 0.0003 - ... - 1.0, 1.0 - 1 hour later `child_work_1` started! - 1 hour later `child_work_2` started! - ... - ----- - -*********** -Next Steps -*********** - -To keep learning about Lightning, build a :ref:`ui_and_frontends`. diff --git a/docs/source-app/code_samples/basics/0.py b/docs/source-app/code_samples/basics/0.py deleted file mode 100644 index 7f4658e3977af..0000000000000 --- a/docs/source-app/code_samples/basics/0.py +++ /dev/null @@ -1,19 +0,0 @@ -from lightning.app import LightningWork - - -class ExampleWork(LightningWork): - def run(self, *args, **kwargs): - print(f"I received the following props: args: {args} kwargs: {kwargs}") - - -work = ExampleWork() -work.run(value=1) - -# Providing the same value. This won't run as already cached. -work.run(value=1) -work.run(value=1) -work.run(value=1) -work.run(value=1) - -# Changing the provided value. This isn't cached and will run again. -work.run(value=10) diff --git a/docs/source-app/code_samples/basics/1.py b/docs/source-app/code_samples/basics/1.py deleted file mode 100644 index 1a696b8e4c45c..0000000000000 --- a/docs/source-app/code_samples/basics/1.py +++ /dev/null @@ -1,22 +0,0 @@ -from lightning.app import LightningWork - - -class ExampleWork(LightningWork): - def __init__(self): - super().__init__(cache_calls=False) - - def run(self, *args, **kwargs): - print(f"I received the following props: args: {args} kwargs: {kwargs}") - - -work = ExampleWork() -work.run(value=1) - -# Providing the same value. This won't run as already cached. -work.run(value=1) -work.run(value=1) -work.run(value=1) -work.run(value=1) - -# Changing the provided value. This isn't cached and will run again. -work.run(value=10) diff --git a/docs/source-app/code_samples/convert_pl_to_app/app.py b/docs/source-app/code_samples/convert_pl_to_app/app.py deleted file mode 100644 index a590cbaab8ea0..0000000000000 --- a/docs/source-app/code_samples/convert_pl_to_app/app.py +++ /dev/null @@ -1,17 +0,0 @@ -from lightning.app import LightningFlow, LightningApp, CloudCompute -from lightning.app.components import TracerPythonScript - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.runner = TracerPythonScript( - "train.py", - cloud_compute=CloudCompute("gpu"), - ) - - def run(self): - self.runner.run() - - -app = LightningApp(RootFlow()) diff --git a/docs/source-app/code_samples/convert_pl_to_app/requirements.txt b/docs/source-app/code_samples/convert_pl_to_app/requirements.txt deleted file mode 100644 index e8fb43ef7dc83..0000000000000 --- a/docs/source-app/code_samples/convert_pl_to_app/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -torch -torchvision -pytorch_lightning diff --git a/docs/source-app/code_samples/convert_pl_to_app/train.py b/docs/source-app/code_samples/convert_pl_to_app/train.py deleted file mode 100644 index d0b7919b75843..0000000000000 --- a/docs/source-app/code_samples/convert_pl_to_app/train.py +++ /dev/null @@ -1,46 +0,0 @@ -import os - -import torch -import torch.nn.functional as F -from torch import nn -from torch.utils.data import DataLoader, random_split -from torchvision import transforms as T -from torchvision.datasets import MNIST - -import lightning.pytorch as pl - - -class LitAutoEncoder(pl.LightningModule): - def __init__(self): - super().__init__() - self.encoder = nn.Sequential(nn.Linear(28 * 28, 128), nn.ReLU(), nn.Linear(128, 3)) - self.decoder = nn.Sequential(nn.Linear(3, 128), nn.ReLU(), nn.Linear(128, 28 * 28)) - - def forward(self, x): - # in lightning, - # forward defines the prediction/inference actions - embedding = self.encoder(x) - return embedding - - def training_step(self, batch, batch_idx): - # training_step defines the train loop. - # It is independent of forward - x, _ = batch - x = x.view(x.size(0), -1) - z = self.encoder(x) - x_hat = self.decoder(z) - loss = F.mse_loss(x_hat, x) - self.log("train_loss", loss) - return loss - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=1e-3) - return optimizer - - -dataset = MNIST(os.getcwd(), download=True, transform=T.ToTensor()) -train, val = random_split(dataset, [55000, 5000]) - -autoencoder = LitAutoEncoder() -trainer = pl.Trainer(accelerator="auto") -trainer.fit(autoencoder, DataLoader(train), DataLoader(val)) diff --git a/docs/source-app/code_samples/quickstart/__init__.py b/docs/source-app/code_samples/quickstart/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/source-app/code_samples/quickstart/app/__init__.py b/docs/source-app/code_samples/quickstart/app/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/source-app/code_samples/quickstart/app/app_0.py b/docs/source-app/code_samples/quickstart/app/app_0.py deleted file mode 100644 index 370f6818a778f..0000000000000 --- a/docs/source-app/code_samples/quickstart/app/app_0.py +++ /dev/null @@ -1,21 +0,0 @@ -from lightning.app import LightningWork, LightningFlow, LightningApp -from docs.quickstart.app_02 import HourLongWork - - -class RootFlow(LightningFlow): - def __init__(self, child_work_1: LightningWork, child_work_2: LightningWork): - super().__init__() - self.child_work_1 = child_work_1 - self.child_work_2 = child_work_2 - - def run(self): - print(round(self.child_work_1.progress, 4), round(self.child_work_2.progress, 4)) - self.child_work_1.run() - self.child_work_2.run() - if self.child_work_1.progress == 1.0: - print("1 hour later `child_work_1` started!") - if self.child_work_2.progress == 1.0: - print("1 hour later `child_work_2` started!") - - -app = LightningApp(RootFlow(HourLongWork(parallel=True), HourLongWork(parallel=True))) diff --git a/docs/source-app/code_samples/quickstart/app/app_1.py b/docs/source-app/code_samples/quickstart/app/app_1.py deleted file mode 100644 index e6b876e452f2b..0000000000000 --- a/docs/source-app/code_samples/quickstart/app/app_1.py +++ /dev/null @@ -1,92 +0,0 @@ -import flash -from flash.core.data.utils import download_data -from flash.image import ImageClassificationData, ImageClassifier - -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute -from lightning.pytorch.callbacks import ModelCheckpoint - - -# Step 1: Create a training LightningWork component that gets a backbone as input -# and saves the best model and its score -class ImageClassifierTrainWork(LightningWork): - def __init__(self, max_epochs: int, backbone: str, cloud_compute: CloudCompute): - # parallel is set to True to run asynchronously - super().__init__(parallel=True, cloud_compute=cloud_compute) - # Number of epochs to run - self.max_epochs = max_epochs - # The model backbone to train on - self.backbone = backbone - self.best_model_path = None - self.best_model_score = None - - def run(self, train_folder): - # Create a datamodule from the given dataset - datamodule = ImageClassificationData.from_folders( - train_folder=train_folder, - batch_size=1, - val_split=0.5, - ) - # Create an image classfier task with the given backbone - model = ImageClassifier(datamodule.num_classes, backbone=self.backbone) - # Start a Lightning trainer, with 1 training batch and 4 validation batches - trainer = flash.Trainer( - max_epochs=self.max_epochs, - limit_train_batches=1, - limit_val_batches=4, - callbacks=[ModelCheckpoint(monitor="val_cross_entropy")], - ) - # Train the model - trainer.fit(model, datamodule=datamodule) - # Save the model path - self.best_model_path = trainer.checkpoint_callback.best_model_path - # Save the model score - self.best_model_score = trainer.checkpoint_callback.best_model_score.item() - - -# Step 2: Create a serving LightningWork component that gets a model input and serves it -class ImageClassifierServeWork(LightningWork): - def run(self, best_model_path: str): - # Load the model from the model path - model = ImageClassifier.load_from_checkpoint(best_model_path) - model.serve(output="labels") - - -# Step 3: Create a root LightningFlow component that gets number of epochs and a path to -# a dataset as inputs, initialize 2 training components and serves the best model -class RootFlow(LightningFlow): - def __init__(self, max_epochs: int, data_dir: str): - super().__init__() - self.data_dir = data_dir - # Init an image classifier with resnet18 backbone - self.train_work_1 = ImageClassifierTrainWork( - max_epochs, - "resnet18", - ) - # Init an image classifier with resnet26 backbone - self.train_work_2 = ImageClassifierTrainWork( - max_epochs, - "resnet26", - ) - # Init the serving component - self.server_work = ImageClassifierServeWork() - - def run(self): - # running both `train_work_1` and `train_work_2` in parallel and asynchronously. - self.train_work_1.run(self.data_dir) - self.train_work_2.run(self.data_dir) - - # run serve_work only when both `best_model_score` are available. - if self.train_work_1.best_model_score and self.train_work_2.best_model_score: - # serve only the best model between `train_work_1` and `train_work_2`. - self.server_work.run( - self.train_work_1.best_model_path - if self.train_work_1.best_model_score < self.train_work_2.best_model_score - else self.train_work_2.best_model_path - ) - - -# Step 4: download a dataset to your local directory under `/data` -download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip", "./data") - -# Initialize your Lightning app with 5 epochs -app = LightningApp(RootFlow(5, "./data/hymenoptera_data")) diff --git a/docs/source-app/code_samples/quickstart/app_01.py b/docs/source-app/code_samples/quickstart/app_01.py deleted file mode 100644 index 42f716f99ffb2..0000000000000 --- a/docs/source-app/code_samples/quickstart/app_01.py +++ /dev/null @@ -1,27 +0,0 @@ -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.utilities.app_helpers import pretty_state - - -class Work(LightningWork): - def __init__(self): - super().__init__(cache_calls=False) - # Attributes are registered automatically in the state. - self.counter = 0 - - def run(self): - # Incrementing an attribute gets reflected in the `Flow` state. - self.counter += 1 - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.w = Work() - - def run(self): - if self.w.has_started: - print(f"State: {pretty_state(self.state)} \n") - self.w.run() - - -app = LightningApp(Flow()) diff --git a/docs/source-app/code_samples/quickstart/app_02.py b/docs/source-app/code_samples/quickstart/app_02.py deleted file mode 100644 index c5c6445a0e32e..0000000000000 --- a/docs/source-app/code_samples/quickstart/app_02.py +++ /dev/null @@ -1,32 +0,0 @@ -from time import sleep - -from lightning.app import LightningWork, LightningFlow, LightningApp - - -# This work takes an hour to run -class HourLongWork(LightningWork): - def __init__(self, parallel: bool = False): - super().__init__(parallel=parallel) - self.progress = 0.0 - - def run(self): - self.progress = 0.0 - for _ in range(3600): - self.progress += 1.0 / 3600 # Reporting my progress to the Flow. - sleep(1) - - -class RootFlow(LightningFlow): - def __init__(self, child_work: LightningWork): - super().__init__() - self.child_work = child_work - - def run(self): - # prints the progress from the child work - print(round(self.child_work.progress, 4)) - self.child_work.run() - if self.child_work.counter == 1.0: - print("1 hour later!") - - -app = LightningApp(RootFlow(HourLongWork())) diff --git a/docs/source-app/code_samples/quickstart/app_03.py b/docs/source-app/code_samples/quickstart/app_03.py deleted file mode 100644 index 6fdbc3ac67083..0000000000000 --- a/docs/source-app/code_samples/quickstart/app_03.py +++ /dev/null @@ -1,31 +0,0 @@ -from time import sleep - -from lightning.app import LightningWork, LightningFlow, LightningApp - - -class HourLongWork(LightningWork): - def __init__(self): - super().__init__(cache_calls=False) - self.progress = 0.0 - - def run(self): - self.progress = 0.0 - for _ in range(3600): - self.progress += 1.0 / 3600 - sleep(1) - - -class RootFlow(LightningFlow): - def __init__(self, child_work: LightningWork): - super().__init__() - self.child_work = child_work - - def run(self): - # prints the progress from the child work - print(round(self.child_work.progress, 4)) - self.child_work.run() - if self.child_work.counter == 1.0: - print("1 hour later!") - - -app = LightningApp(RootFlow(HourLongWork())) diff --git a/docs/source-app/code_samples/quickstart/app_comp.py b/docs/source-app/code_samples/quickstart/app_comp.py deleted file mode 100644 index 9aee70009f478..0000000000000 --- a/docs/source-app/code_samples/quickstart/app_comp.py +++ /dev/null @@ -1,26 +0,0 @@ -from lightning.app import LightningFlow, LightningApp -from lightning.app.testing import EmptyFlow, EmptyWork - - -class FlowB(LightningFlow): - def __init__(self): - super().__init__() - self.flow_d = EmptyFlow() - self.work_b = EmptyWork() - - def run(self): - ... - - -class FlowA(LightningFlow): - def __init__(self): - super().__init__() - self.flow_b = FlowB() - self.flow_c = EmptyFlow() - self.work_a = EmptyWork() - - def run(self): - ... - - -app = LightningApp(FlowA()) diff --git a/docs/source-app/code_samples/quickstart/hello_world/app.py b/docs/source-app/code_samples/quickstart/hello_world/app.py deleted file mode 100644 index 18dd2d78a0c6f..0000000000000 --- a/docs/source-app/code_samples/quickstart/hello_world/app.py +++ /dev/null @@ -1,15 +0,0 @@ -from lightning.app import LightningFlow, LightningApp - - -# Step 1: Subclass LightningFlow component to define the app flow. -class HelloWorld(LightningFlow): - # Step 2: Add the app logic to the LightningFlow run method to - # ``print("Hello World!")`. - # The LightningApp executes the run method of the main LightningFlow - # within an infinite loop. - def run(self): - print("Hello World!") - - -# Step 3: Initialize a LightningApp with the LightningFlow you defined (in step 1) -app = LightningApp(HelloWorld()) diff --git a/docs/source-app/code_samples/quickstart/hello_world/app_ui.py b/docs/source-app/code_samples/quickstart/hello_world/app_ui.py deleted file mode 100644 index ad0e5065f5bb5..0000000000000 --- a/docs/source-app/code_samples/quickstart/hello_world/app_ui.py +++ /dev/null @@ -1,57 +0,0 @@ -import os - -from lightning.app import LightningFlow, LightningApp -from lightning.app.frontend import StaticWebFrontend, StreamlitFrontend -from lightning.app.utilities.state import AppState - - -# Step 1: Define your LightningFlow component with the app UI -class UIStreamLit(LightningFlow): - def __init__(self): - super().__init__() - self.should_print = False - - # Step 2: Override `configure_layout` to define the layout of the UI - # In this example, we are using `StreamlitFrontend` - def configure_layout(self): - return StreamlitFrontend(render_fn=render_fn) - - -# Step 3: Implement the StreamLit render method -def render_fn(state: AppState): - import streamlit as st - from streamlit_autorefresh import st_autorefresh - - st_autorefresh(interval=2000, limit=None, key="refresh") - - state.should_print = st.select_slider( - "Should the Application print 'Hello World !' to the terminal:", - [False, True], - ) - - -# Step 4: Implement a Static Web Frontend. This could be react, vue, etc. -class UIStatic(LightningFlow): - # Step 5: - def configure_layout(self): - return StaticWebFrontend(os.path.join(os.path.dirname(__file__), "ui")) - - -# Step 6: Implement the root flow. -class HelloWorld(LightningFlow): - def __init__(self): - super().__init__() - self.static_ui = UIStatic() - self.streamlit_ui = UIStreamLit() - - def run(self): - print("Hello World!" if self.streamlit_ui.should_print else "") - - def configure_layout(self): - return [ - {"name": "StreamLit", "content": self.streamlit_ui}, - {"name": "Static", "content": self.static_ui}, - ] - - -app = LightningApp(HelloWorld()) diff --git a/docs/source-app/code_samples/quickstart/hello_world/ui/index.html b/docs/source-app/code_samples/quickstart/hello_world/ui/index.html deleted file mode 100644 index fe38c432f504c..0000000000000 --- a/docs/source-app/code_samples/quickstart/hello_world/ui/index.html +++ /dev/null @@ -1 +0,0 @@ -
Hello from component UIStatic
diff --git a/docs/source-app/conf.py b/docs/source-app/conf.py deleted file mode 100644 index 5399d8205cd49..0000000000000 --- a/docs/source-app/conf.py +++ /dev/null @@ -1,412 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - -import glob -import inspect -import os -import shutil -import sys - -import lai_sphinx_theme -from lightning_utilities.docs import fetch_external_assets - -import lightning - -_PATH_HERE = os.path.abspath(os.path.dirname(__file__)) -_PATH_ROOT = os.path.realpath(os.path.join(_PATH_HERE, "..", "..")) -sys.path.insert(0, os.path.abspath(_PATH_ROOT)) - -_SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) -_FAST_DOCS_DEV = int(os.environ.get("FAST_DOCS_DEV", True)) -_FETCH_S3_ASSETS = int(os.getenv("DOCS_FETCH_ASSETS", not _FAST_DOCS_DEV)) - -# -- Project information ----------------------------------------------------- - -# this name shall match the project name in Github as it is used for linking to code -project = "lightning" -copyright = lightning.__copyright__ -author = lightning.__author__ - -# The short X.Y version -version = lightning.__version__ -# The full version, including alpha/beta/rc tags -release = lightning.__version__ - -# -- Project documents ------------------------------------------------------- - -if _FETCH_S3_ASSETS: - fetch_external_assets( - docs_folder=_PATH_HERE, - assets_folder="_static/fetched-s3-assets", - retrieve_pattern=r"https?://[-a-zA-Z0-9_]+\.s3\.[-a-zA-Z0-9()_\\+.\\/=]+" - ) - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. - -needs_sphinx = "5.3" - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.doctest", - "sphinx.ext.intersphinx", - "sphinx_toolbox.collapse", - "sphinx.ext.todo", - "sphinx.ext.coverage", - # "sphinx.ext.linkcode", - "sphinx.ext.autosummary", - "sphinx.ext.napoleon", - # 'sphinxcontrib.mockautodoc', # raises error: directive 'automodule' is already registered ... - # 'sphinxcontrib.fulltoc', # breaks pytorch-theme with unexpected kw argument 'titles_only' - "sphinxcontrib.video", - "myst_parser", - "sphinx.ext.autosectionlabel", - "nbsphinx", - "sphinx_autodoc_typehints", - "sphinx_copybutton", - "sphinx_paramlinks", - "sphinx_togglebutton", - "sphinx.ext.githubpages", - "lai_sphinx_theme.extensions.lightning", - 'sphinx.ext.mathjax', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# myst-parser, forcing to parse all html pages with mathjax -# https://github.com/executablebooks/MyST-Parser/issues/394 -myst_update_mathjax = False -# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html?highlight=anchor#auto-generated-header-anchors -myst_heading_anchors = 3 - -# https://berkeley-stat159-f17.github.io/stat159-f17/lectures/14-sphinx..html#conf.py-(cont.) -# https://stackoverflow.com/questions/38526888/embed-ipython-notebook-in-sphinx-document -# I execute the notebooks manually in advance. If notebooks test the code, -# they should be run at build time. -nbsphinx_execute = "never" -nbsphinx_allow_errors = True -nbsphinx_requirejs_path = "" - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -# source_suffix = ['.rst', '.md', '.ipynb'] -source_suffix = { - ".rst": "restructuredtext", - ".txt": "markdown", - ".md": "markdown", - ".ipynb": "nbsphinx", -} - -# The master toctree document. -master_doc = "index" - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = 'en' - -# List of patterns, relative to source-app directory, that match files and -# directories to ignore when looking for source-app files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [ - "PULL_REQUEST_TEMPLATE.md", - "**/README.md/*", - "readme.md", - "_templates", - "code_samples/convert_pl_to_app/requirements.txt", - "**/_static/*" -] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "lai_sphinx_theme" -html_theme_path = [os.environ.get('LIT_SPHINX_PATH', lai_sphinx_theme.get_html_theme_path())] - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. - -html_theme_options = { - "pytorch_project": lightning.__homepage__, - "analytics_id": "G-D3Q2ESCTZR", - "canonical_url": lightning.__homepage__, - "collapse_navigation": False, - "display_version": True, - "logo_only": False, -} - -html_favicon = "_static/images/icon.svg" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_templates", "_static"] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = project + "-doc" - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # 'preamble': '', - # Latex figure (float) alignment - "figure_align": "htbp", -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source-app start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, project + ".tex", project + " Documentation", author, "manual"), -] - -# MathJax configuration -mathjax3_config = { - 'tex': { - 'packages': {'[+]': ['ams', 'newcommand', 'configMacros']} - }, -} - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source-app start file, name, description, authors, manual section). -man_pages = [(master_doc, project, project + " Documentation", [author], 1)] - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source-app start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - project, - project + " Documentation", - author, - project, - lightning.__docs__, - "Miscellaneous", - ), -] - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ["search.html"] - -# -- Extension configuration ------------------------------------------------- - -# -- Options for intersphinx extension --------------------------------------- - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "torch": ("https://pytorch.org/docs/stable/", None), - "numpy": ("https://numpy.org/doc/stable/", None), -} - -nitpicky = True - - -nitpick_ignore = [ - ("py:class", "typing.Self"), - # missing in generated API - ("py:exc", "MisconfigurationException"), - # TODO: generated list of all existing ATM, need to be fixed - ('py:exc', 'ApiException'), - ('py:class', 'BaseModel'), - ('py:exc', 'LightningPlatformException'), - ('py:class', 'forwarded'), - ('py:class', 'lightning.app.api.http_methods.Delete'), - ('py:class', 'lightning.app.api.http_methods.Get'), - ('py:class', 'lightning.app.api.http_methods.HttpMethod'), - ('py:class', 'lightning.app.api.http_methods.Post'), - ('py:class', 'lightning.app.api.http_methods.Put'), - ('py:class', 'lightning.app.components.python.TracerPythonScript'), - ('py:func', 'lightning.app.pdb.set_trace'), - ('py:class', 'lightning.app.runners.runtime.Runtime'), - ('py:class', 'lightning.app.source_code.local.LocalSourceCodeDir'), - ('py:class', 'lightning.app.storage.payload._BasePayload'), - ('py:class', 'lightning.app.structures.Dict'), - ('py:class', 'lightning.app.structures.List'), - ('py:class', 'lightning.app.testing.testing.LightningTestApp'), - ('py:class', 'lightning.app.utilities.app_status.WorkStatus'), - ('py:class', 'lightning.app.utilities.frontend.AppInfo'), - ('py:class', 'lightning.app.utilities.packaging.app_config.AppConfig'), - ('py:class', 'lightning.app.utilities.packaging.build_config.BuildConfig'), - ('py:class', 'lightning.app.utilities.packaging.cloud_compute.CloudCompute'), - ('py:class', 'lightning.app.utilities.proxies.WorkRunExecutor'), - ('py:class', 'lightning.app.utilities.tracer.Tracer'), - ('py:class', 'lightning_cloud.openapi.models.cloudspace_id_runs_body.CloudspaceIdRunsBody'), - ('py:class', 'lightning_cloud.openapi.models.externalv1_lightningapp_instance.Externalv1LightningappInstance'), - ('py:class', 'lightning_cloud.openapi.models.v1_cloud_space.V1CloudSpace'), - ('py:class', 'lightning_cloud.openapi.models.v1_env_var.V1EnvVar'), - ('py:class', 'lightning_cloud.openapi.models.v1_flowserver.V1Flowserver'), - ('py:class', 'lightning_cloud.openapi.models.v1_lightning_auth.V1LightningAuth'), - ('py:class', 'lightning_cloud.openapi.models.v1_lightning_run.V1LightningRun'), - ('py:class', 'lightning_cloud.openapi.models.v1_lightningwork_drives.V1LightningworkDrives'), - ('py:class', 'lightning_cloud.openapi.models.v1_membership.V1Membership'), - ('py:class', 'lightning_cloud.openapi.models.v1_network_config.V1NetworkConfig'), - ('py:class', 'lightning_cloud.openapi.models.v1_queue_server_type.V1QueueServerType'), - ('py:class', 'lightning_cloud.openapi.models.v1_work.V1Work'), - ('py:class', 'pydantic.main.BaseModel'), - ('py:meth', 'transfer'), -] - -# -- Options for todo extension ---------------------------------------------- - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True - - -def setup(app): - # this is for hiding doctest decoration, - # see: http://z4r.github.io/python/2011/12/02/hides-the-prompts-and-output/ - app.add_js_file("copybutton.js") - app.add_css_file("main.css") - -# copy all examples to local folder -path_examples = os.path.join(_PATH_HERE, "..", "examples") -if not os.path.isdir(path_examples): - os.mkdir(path_examples) -for path_app_example in glob.glob(os.path.join(_PATH_ROOT, "examples", "app_*")): - path_app_example2 = os.path.join(path_examples, os.path.basename(path_app_example)) - if not os.path.isdir(path_app_example2): - shutil.copytree(path_app_example, path_app_example2, dirs_exist_ok=True) - - -# Ignoring Third-party packages -# https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule -def _package_list_from_file(file): - list_pkgs = [] - with open(file) as fp: - lines = fp.readlines() - for ln in lines: - found = [ln.index(ch) for ch in list(",=<>#") if ch in ln] - pkg = ln[: min(found)] if found else ln - if pkg.rstrip(): - list_pkgs.append(pkg.rstrip()) - return list_pkgs - - -# define mapping from PyPI names to python imports -PACKAGE_MAPPING = { - "PyYAML": "yaml", -} -MOCK_PACKAGES = [] -if _SPHINX_MOCK_REQUIREMENTS: - # mock also base packages when we are on RTD since we don't install them there - MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_ROOT, "requirements.txt")) -MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES] - -autodoc_mock_imports = MOCK_PACKAGES - - -autosummary_generate = True - -autodoc_member_order = "groupwise" -autoclass_content = "both" -# the options are fixed and will be soon in release, -# see https://github.com/sphinx-doc/sphinx/issues/5459 -autodoc_default_options = { - "members": None, - "methods": None, - # 'attributes': None, - "special-members": "__call__", - "exclude-members": "_abc_impl", - "show-inheritance": True, - "private-members": True, - "noindex": True, -} - -# Sphinx will add “permalinks” for each heading and description environment as paragraph signs that -# become visible when the mouse hovers over them. -# This value determines the text for the permalink; it defaults to "¶". Set it to None or the empty -# string to disable permalinks. -# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_permalinks -# html_add_permalinks = "¶" -# True to prefix each section label with the name of the document it is in, followed by a colon. -# For example, index:Introduction for a section called Introduction that appears in document index.rst. -# Useful for avoiding ambiguity when the same section heading appears in different documents. -# http://www.sphinx-doc.org/en/master/usage/extensions/autosectionlabel.html -autosectionlabel_prefix_document = True - -# only run doctests marked with a ".. doctest::" directive -doctest_test_doctest_blocks = "" -doctest_global_setup = """ -import importlib -import os - -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute -from lightning.fabric.loggers.tensorboard import _TENSORBOARD_AVAILABLE, _TENSORBOARDX_AVAILABLE -""" -coverage_skip_undoc_in_source = True - -# skip false positive linkcheck errors from anchors -linkcheck_anchors = False - -# A timeout value, in seconds, for the linkcheck builder. -linkcheck_timeout = 60 - -# ignore all links in any CHANGELOG file -linkcheck_exclude_documents = [r"^(.*\/)*CHANGELOG.*$"] - - -# ignore the following relative links (false positive errors during linkcheck) -linkcheck_ignore = [ - "https://www.openai.com/index/clip/", -] diff --git a/docs/source-app/contribute_app.rst b/docs/source-app/contribute_app.rst deleted file mode 100644 index 2f690e8479062..0000000000000 --- a/docs/source-app/contribute_app.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -################# -Contribute an app -################# - -Show off your work! Contribute and example to be highlighted in our documentation and App gallery. diff --git a/docs/source-app/core_api/lightning_app/app.py b/docs/source-app/core_api/lightning_app/app.py deleted file mode 100644 index 42f716f99ffb2..0000000000000 --- a/docs/source-app/core_api/lightning_app/app.py +++ /dev/null @@ -1,27 +0,0 @@ -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.utilities.app_helpers import pretty_state - - -class Work(LightningWork): - def __init__(self): - super().__init__(cache_calls=False) - # Attributes are registered automatically in the state. - self.counter = 0 - - def run(self): - # Incrementing an attribute gets reflected in the `Flow` state. - self.counter += 1 - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.w = Work() - - def run(self): - if self.w.has_started: - print(f"State: {pretty_state(self.state)} \n") - self.w.run() - - -app = LightningApp(Flow()) diff --git a/docs/source-app/core_api/lightning_app/communication.rst b/docs/source-app/core_api/lightning_app/communication.rst deleted file mode 100644 index d1aa1d35f54b1..0000000000000 --- a/docs/source-app/core_api/lightning_app/communication.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -########################################## -Communication between Lightning Components -########################################## - -**Audience:** Users that want to create interactive applications. - -**Level:** Intermediate - -**Prerequisite**: Read the :doc:`Communication in Lightning Apps article <../../../workflows/access_app_state>`. - ----- - -.. include:: ../../core_api/lightning_app/communication_content.rst diff --git a/docs/source-app/core_api/lightning_app/communication_content.rst b/docs/source-app/core_api/lightning_app/communication_content.rst deleted file mode 100644 index 4b373dbb07588..0000000000000 --- a/docs/source-app/core_api/lightning_app/communication_content.rst +++ /dev/null @@ -1,160 +0,0 @@ - -******************************** -Communication Between Components -******************************** - -When creating interactive Lightning Apps (App) with multiple components, you may need your components to share information with each other and rely on that information to control their execution, share progress in the UI, trigger a sequence of operations, etc. - -To accomplish that, Lightning components can communicate using the App State. The App State is composed of all attributes defined within each component's **__init__** method e.g anything attached to the component with **self.x = y**. - -All attributes of all **LightningWork (Work)** components are accessible in the **LightningFlow (Flow)** components in real-time. - -By design, the Flows communicate to all **Works** within the application. However, Works can't communicate with each other directly, they must use Flows as a proxy to communicate. - -Once a Work is running, any updates to the Work's state is automatically communicated to the Flow, as a delta (using `DeepDiff `_). The state communication isn't bi-directional, communication is only done from Work to Flow. - -Internally, the App is alternatively collecting deltas sent from all the registered Works and/or UI, and running the root Flow run method of the App. - ----- - -************************************************* -Communication from LightningWork to LightningFlow -************************************************* - -LightningFlow (Flow) can access their children's LightningWork (Work) state. - -When a running Work attribute gets updated inside its method (separate process locally or remote machine), the app re-executes Flow's run method once it receives the state update from the Work. - -Here's an example to better understand communication from Work to Flow. - -The ``WorkCounter`` increments a counter until 1 million and the ``Flow`` prints the work counter. - -As the Work is running its own process, its state changes are sent to the Flow which contains the latest value of the counter. - -.. code-block:: python - - import lightning as L - - - class WorkCounter(L.LightningWork): - def __init__(self): - super().__init__(parallel=True) - self.counter = 0 - - def run(self): - for _ in range(int(10e6)): - self.counter += 1 - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.w = WorkCounter() - - def run(self): - self.w.run() - print(self.w.counter) - - - app = L.LightningApp(Flow()) - - -A delta sent from the Work to the Flow looks like this: - -.. code-block:: python - - {"values_changed": {"root['works']['w']['vars']['counter']": {"new_value": 425}}} - -Here is the associated illustration: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/deltas.gif - :alt: Mechanism showing how delta are sent. - :width: 100 % - -Here's another example that is slightly different. Here we define a Flow and Work, where the Work increments a counter indefinitely and the Flow prints its state which contain the Work. - -You can easily check the state of your entire app as follows: - -.. literalinclude:: ../../core_api/lightning_app/app.py - -Run the app with: - -.. code-block:: bash - - lightning run app docs/source/core_api/lightning_app/app.py - -And here's the output you get when running the App using the **Lightning CLI**: - -.. code-block:: console - - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - State: {'works': {'w': {'vars': {'counter': 1}}}} - State: {'works': {'w': {'vars': {'counter': 2}}}} - State: {'works': {'w': {'vars': {'counter': 3}}}} - State: {'works': {'w': {'vars': {'counter': 3}}}} - State: {'works': {'w': {'vars': {'counter': 4}}}} - ... - ----- - -************************************************* -Communication from LightningFlow to LightningWork -************************************************* - -Communication from the LightningFlow (Flow) to the LightningWork (Work) while running **isn't supported yet**. If your application requires this feature, please open an issue on Github. - -Here's an example of what would happen if you try to have the Flow communicate with the Work: - -.. code-block:: python - - import lightning as L - from time import sleep - - - class WorkCounter(L.LightningWork): - def __init__(self): - super().__init__(parallel=True) - self.counter = 0 - - def run(self): - while True: - sleep(1) - print(f"Work {self.counter}") - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.w = WorkCounter() - - def run(self): - self.w.run() - sleep(1) - print(f"Flow {self.w.counter}") - self.w.counter += 1 - - - app = L.LightningApp(Flow()) - -As you can see, there is a divergence between the values within the Work and the Flow. - -.. code-block:: console - - Flow 0 - Flow 1 - Flow 2 - Flow 3 - Work 0 - Flow 4 - Work 0 - Flow 5 - Work 0 - Flow 6 - Work 0 - Flow 7 - Work 0 - Flow 8 - Work 0 - Flow 9 - Work 0 - Flow 10 diff --git a/docs/source-app/core_api/lightning_app/compute_content.rst b/docs/source-app/core_api/lightning_app/compute_content.rst deleted file mode 100644 index 8bb2e7039ad83..0000000000000 --- a/docs/source-app/core_api/lightning_app/compute_content.rst +++ /dev/null @@ -1,40 +0,0 @@ -:orphan: - -*************************** -Customize my Flow resources -*************************** - -In the cloud, you can simply configure which machine to run on by passing -a :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute` to your work ``__init__`` method: - -.. code-block:: python - - import lightning as L - - # Run on a small, shared CPU machine. This is the default for every LightningFlow. - app = L.LightningApp(L.Flow(), flow_cloud_compute=L.CloudCompute()) - - -Here is the full list of supported machine names: - -.. list-table:: Hardware by Accelerator Type - :widths: 25 25 25 - :header-rows: 1 - - * - Name - - # of CPUs - - Memory - * - flow-lite - - 0.3 - - 4 GB - -The up-to-date prices for these instances can be found `here `_. - ----- - -************ -CloudCompute -************ - -.. autoclass:: lightning.app.utilities.packaging.cloud_compute.CloudCompute - :noindex: diff --git a/docs/source-app/core_api/lightning_app/dynamic_work.rst b/docs/source-app/core_api/lightning_app/dynamic_work.rst deleted file mode 100644 index bf202aa590a79..0000000000000 --- a/docs/source-app/core_api/lightning_app/dynamic_work.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -.. _dynamic_work: - -##################### -Dynamic LightningWork -##################### - -**Audience:** Users who want to create applications that adapt to user demands. - -**Level:** Advanced - ----- - -.. include:: dynamic_work_content.rst diff --git a/docs/source-app/core_api/lightning_app/dynamic_work_content.rst b/docs/source-app/core_api/lightning_app/dynamic_work_content.rst deleted file mode 100644 index 31616a42a1493..0000000000000 --- a/docs/source-app/core_api/lightning_app/dynamic_work_content.rst +++ /dev/null @@ -1,202 +0,0 @@ -*************************************** -What Dynamic LightningWork does for you -*************************************** - -Dynamic LightningWork (Work) changes the resources your application uses while the application is running (aka at runtime). - -For example, imagine you want to create a research notebook app for your team. You want every member to be able to create multiple `JupyterLab `_ sessions on their hardware of choice. - -To allow every notebook to choose hardware, it needs to be set up in it's own :class:`~lightning.app.core.work.LightningWork`, but you can't know the number of notebooks user will need in advance. In this case you'll need to add ``LightningWorks`` dynamically at run time. - ----- - -***************** -Use Dynamic Works -***************** - -Dynamic Works should be used anytime you want change the resources your application is using while it is running (aka at runtime). - -You're usually going to use the ``start`` and ``stop`` methods together. - ----- - -Add a Dynamic Work -^^^^^^^^^^^^^^^^^^ - -There are a couple of ways you can add a dynamic Work: - -- Option 1: Attach your components in the **run** method using the Python functions. -- Option 2: Use the Lightning built-in classes :class:`~lightning.app.structures.Dict` or :class:`~lightning.app.structures.List`. - -.. note:: Using the Lightning built-in classes is usually easier to read. - ----- - -**OPTION 1:** Attach your components in the run method of a flow using the Python functions **hasattr**, **setattr**, and **getattr**: - -.. code-block:: python - - class RootFlow(lapp.LightningFlow): - - def run(self): - - if not hasattr(self, "work"): - # The `Work` component is created and attached here. - setattr(self, "work", Work()) - # Run the `Work` component. - getattr(self, "work").run() - -**OPTION 2:** Use the built-in Lightning classes :class:`~lightning.app.structures.Dict` or :class:`~lightning.app.structures.List` - -.. code-block:: python - - from lightning.app.structures import Dict - - class RootFlow(lapp.LightningFlow): - - def __init__(self): - super().__init__() - self.dict = Dict() - - def run(self): - if "work" not in self.dict: - # The `Work` component is attached here. - self.dict["work"] = Work() - self.dict["work"].run() - ----- - -Stop a Work -^^^^^^^^^^^ -Stop a work when you are concerned about cost. - -To stop a work, use the work ``stop`` method: - -.. code-block:: python - - class RootFlow(L.LightningFlow): - - def __init__(self): - super().__init__() - self.work = Work() - - def run(self): - self.work.stop() - ----- - -********************* -Dynamic Work Examples -********************* - -.. - The entire application can be found `here `_. - ----- - -Dynamic Work with Jupyter Notebooks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this example, we are dynamically creating ``JupyterLabWork`` every time a user clicks the **Create Jupyter Notebook** button. - -In order to do that, we are iterating over the list of ``jupyter_config_requests`` infinitely. - -.. code-block:: python - - import lightning as L - - - class JupyterLabManager(L.LightningFlow): - - """This flow manages the users notebooks running within works."""" - - def __init__(self): - super().__init__() - self.jupyter_works = L.structures.Dict() - self.jupyter_config_requests = [] - - def run(self): - for idx, jupyter_config in enumerate(self.jupyter_config_requests): - - # The Jupyter Config has this form is: - # {"use_gpu": False/True, "token": None, "username": ..., "stop": False} - - # Step 1: Check if JupyterWork already exists for this username - username = jupyter_config["username"] - if username not in self.jupyter_works: - jupyter_config["ready"] = False - - # Set the hardware selected by the user: GPU or CPU. - cloud_compute = L.CloudCompute("gpu" if jupyter_config["use_gpu"] else "cpu-small") - - # Step 2: Create new JupyterWork dynamically ! - self.jupyter_works[username] = JupyterLabWork(cloud_compute=cloud_compute) - - # Step 3: Run the JupyterWork - self.jupyter_works[username].run() - - # Step 4: Store the notebook token in the associated config. - # We are using this to know when the notebook is ready - # and display the stop button on the UI. - if self.jupyter_works[username].token: - jupyter_config["token"] = self.jupyter_works[username].token - - # Step 5: Stop the work if the user requested it. - if jupyter_config['stop']: - self.jupyter_works[username].stop() - self.jupyter_config_requests.pop(idx) - - def configure_layout(self): - return L.app.frontend.StreamlitFrontend(render_fn=render_fn) - ----- - -Dynamic Works with StreamLit UI -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Continuing from the Jupyter Notebook example, in the UI, we receive the **state** of the Jupyter Manager and the state can be modified directly from the UI. - -.. code-block:: python - - import streamlit as st - - - def render_fn(state): - - # Step 1: Enable users to select their notebooks and create them - column_1, column_2, column_3 = st.columns(3) - with column_1: - create_jupyter = st.button("Create Jupyter Notebook") - with column_2: - username = st.text_input('Enter your username', "tchaton") - assert username - with column_3: - use_gpu = st.checkbox('Use GPU') - - # Step 2: If a user clicked the button, add an element to the list of configs - # Note: state.jupyter_config_requests = ... will sent the state update to the component. - if create_jupyter: - new_config = [{"use_gpu": use_gpu, "token": None, "username": username, "stop": False}] - state.jupyter_config_requests = state.jupyter_config_requests + new_config - - # Step 3: List of running notebooks. - for idx, config in enumerate(state.jupyter_config_requests): - column_1, column_2, column_3 = st.columns(3) - with column_1: - if not idx: - st.write(f"Idx") - st.write(f"{idx}") - with column_2: - if not idx: - st.write(f"Use GPU") - st.write(config['use_gpu']) - with column_3: - if not idx: - st.write(f"Stop") - if config["token"]: - should_stop = st.button("Stop this notebook") - - # Step 4: Change stop if the user clicked the button - if should_stop: - config["stop"] = should_stop - state.jupyter_config_requests = state.jupyter_config_requests diff --git a/docs/source-app/core_api/lightning_app/index.rst b/docs/source-app/core_api/lightning_app/index.rst deleted file mode 100644 index bf0430f3bee5b..0000000000000 --- a/docs/source-app/core_api/lightning_app/index.rst +++ /dev/null @@ -1,94 +0,0 @@ -############# -Lightning App -############# -**Audience:** Users who want to know how an app works under the hood 🤯. - -**Lightning App:** We call workflows composed of multiple LightningWorks a **Lightning App**. - ----- - -******************* -Peek under the hood -******************* - - -.. raw:: html - -
-
- -.. displayitem:: - :header: App Components Tree (Basic) - :description: Learn more component composition and nesting. - :col_css: col-md-4 - :button_link: ../../glossary/app_tree.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: The event loop (Basic) - :description: Learn more about the event loop. - :col_css: col-md-4 - :button_link: ../../glossary/event_loop.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Communication between Flow and Works - :description: Learn more about components communicate. - :col_css: col-md-4 - :button_link: communication.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Customize Flow compute resources - :description: Learn more about Flow customizations. - :col_css: col-md-4 - :button_link: compute_content.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Dynamically create, execute and stop Work - :description: Learn more about components creation. - :col_css: col-md-4 - :button_link: dynamic_work.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Sharing My Components (Intermediate) - :description: Learn more component composition and nesting. - :col_css: col-md-4 - :button_link: ../../glossary/sharing_components.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
- ----- - -***************** -Lightning App API -***************** - -.. raw:: html - -
-
- -.. displayitem:: - :header: LightningApp API - :description: Look into the Lightning API reference. - :col_css: col-md-4 - :button_link: lightning_app.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/core_api/lightning_app/lightning_app.rst b/docs/source-app/core_api/lightning_app/lightning_app.rst deleted file mode 100644 index af9592628a6f2..0000000000000 --- a/docs/source-app/core_api/lightning_app/lightning_app.rst +++ /dev/null @@ -1,12 +0,0 @@ -:orphan: - -.. _lightning_app: - -############ -LightningApp -############ - - -.. autoclass:: lightning.app.core.app.LightningApp - :exclude-members: _run, connect, get_component_by_name, maybe_apply_changes, set_state - :noindex: diff --git a/docs/source-app/core_api/lightning_flow.rst b/docs/source-app/core_api/lightning_flow.rst deleted file mode 100644 index 642112ae02793..0000000000000 --- a/docs/source-app/core_api/lightning_flow.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _lightning_flow: - -############# -LightningFlow -############# - -.. autoclass:: lightning.app.core.flow.LightningFlow - :exclude-members: _attach_backend, _exit, _is_state_attribute, set_state diff --git a/docs/source-app/core_api/lightning_work/compute.rst b/docs/source-app/core_api/lightning_work/compute.rst deleted file mode 100644 index 89313c4878cec..0000000000000 --- a/docs/source-app/core_api/lightning_work/compute.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -.. _cloud_compute: - -############################ -Customize your Cloud Compute -############################ - -**Audience:** Users who want to select the hardware to run in the cloud. - -**Level:** Intermediate - ----- - -.. include:: compute_content.rst diff --git a/docs/source-app/core_api/lightning_work/compute_content.rst b/docs/source-app/core_api/lightning_work/compute_content.rst deleted file mode 100644 index 1ca64429dc342..0000000000000 --- a/docs/source-app/core_api/lightning_work/compute_content.rst +++ /dev/null @@ -1,94 +0,0 @@ - -*************************** -Customize my Work resources -*************************** - -In the cloud, you can simply configure which machine to run on by passing -a :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute` to your work ``__init__`` method: - -.. code-block:: python - - import lightning as L - - # Run on a free, shared CPU machine. This is the default for every LightningWork. - MyCustomWork(cloud_compute=L.CloudCompute()) - - # Run on a dedicated, medium-size CPU machine (see specs below) - MyCustomWork(cloud_compute=L.CloudCompute("cpu-medium")) - - # Run on cheap GPU machine with a single GPU (see specs below) - MyCustomWork(cloud_compute=L.CloudCompute("gpu")) - - # Run on a fast multi-GPU machine (see specs below) - MyCustomWork(cloud_compute=L.CloudCompute("gpu-fast-multi")) - -.. warning:: - Custom base images are not supported with the default CPU cloud compute. For example: - - .. code-block:: py - - class MyWork(LightningWork): - def __init__(self): - super().__init__(cloud_build_config=BuildConfig(image="my-custom-image")) # no cloud compute, for example default work - - -Here is the full list of supported machine names: - -.. list-table:: Hardware by Accelerator Type - :widths: 25 25 25 25 - :header-rows: 1 - - * - Name - - # of CPUs - - GPUs - - Memory - * - default - - 1 - - 0 - - 4 GB - * - cpu-small - - 2 - - 0 - - 8 GB - * - cpu-medium - - 8 - - 0 - - 32 GB - * - gpu - - 4 - - 1 (T4, 16 GB) - - 16 GB - * - gpu-fast - - 8 - - 1 (V100, 16 GB) - - 61 GB - * - gpu-fast-multi - - 32 - - 4 (V100 16 GB) - - 244 GB - -The up-to-date prices for these instances can be found `here `_. - ----- - -********************** -Stop my work when idle -********************** - -By providing **idle_timeout=X Seconds**, the work is automatically stopped **X seconds** after doing nothing. - -.. code-block:: python - - import lightning as L - - # Run on a single CPU and turn down immediately when idle. - MyCustomWork(cloud_compute=L.CloudCompute("gpu", idle_timeout=0)) - ----- - -************ -CloudCompute -************ - -.. autoclass:: lightning.app.utilities.packaging.cloud_compute.CloudCompute - :noindex: diff --git a/docs/source-app/core_api/lightning_work/handling_app_exception.rst b/docs/source-app/core_api/lightning_work/handling_app_exception.rst deleted file mode 100644 index 20c9b618d97aa..0000000000000 --- a/docs/source-app/core_api/lightning_work/handling_app_exception.rst +++ /dev/null @@ -1,13 +0,0 @@ -:orphan: - -############################### -Handle Lightning App exceptions -############################### - -**Audience:** Users who want to make Lightning Apps more robust to potential issues. - -**Level:** Advanced - ----- - -.. include:: handling_app_exception_content.rst diff --git a/docs/source-app/core_api/lightning_work/handling_app_exception_content.rst b/docs/source-app/core_api/lightning_work/handling_app_exception_content.rst deleted file mode 100644 index 4840cf5fdf6f3..0000000000000 --- a/docs/source-app/core_api/lightning_work/handling_app_exception_content.rst +++ /dev/null @@ -1,74 +0,0 @@ - -*************************************************** -What handling Lightning App exceptions does for you -*************************************************** - -Imagine you are creating a Lightning App (App) where your team can launch model training by providing their own Github Repo any time they want. - -As the App admin, you don't want the App to go down if their code has a bug and breaks. - -Instead, you would like the LightningWork (Work) to capture the exception and present the issue to users. - ----- - -**************************** -Configure exception handling -**************************** - -The LightningWork (Work) accepts an argument **raise_exception** which is **True** by default. This aligns with Python default behaviors. - -However, for the user case stated in the previous section, we want to capture the Work exceptions. This is done by providing ``raise_exception=False`` to the work ``__init__`` method. - -.. code-block:: python - - import lightning as L - - MyCustomWork(raise_exception=False) # <== HERE: The exception is captured. - - # Default behavior - MyCustomWork(raise_exception=True) # <== HERE: The exception is raised within the flow and terminates the app - - -And you can customize this behavior by overriding the ``on_exception`` hook to the Work. - -.. code-block:: python - - import lightning as L - - class MyCustomWork(L.LightningWork): - - def on_exception(self, exception: Exception): - # do something when an exception is triggered. - ----- - -************************** -Exception handling example -************************** - -This is the pseudo-code for the application described above. - -.. code-block:: python - - import lightning as L - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.user_jobs = L.structures.Dict() - self.requested_jobs = [] - - def run(self): - for request in self.requested_jobs: - job_id = request["id"] - if job_id not in self.user_jobs: - # Note: The `GithubRepoLauncher` doesn't exist yet. - self.user_jobs[job_id] = GithubRepoLauncher( - **request, - raise_exception=False, # <== HERE: The exception is captured. - ) - self.user_jobs[job_id].run() - - if self.user_jobs[job_id].status.stage == "failed" and "printed" not in request: - print(self.user_jobs[job_id].status) # <== HERE: Print the user exception. - request["printed"] = True diff --git a/docs/source-app/core_api/lightning_work/index.rst b/docs/source-app/core_api/lightning_work/index.rst deleted file mode 100644 index 0b660f2209aba..0000000000000 --- a/docs/source-app/core_api/lightning_work/index.rst +++ /dev/null @@ -1,112 +0,0 @@ -############## -Lightning Work -############## - -**Audience:** Users who want to know how Lightning Work works under the hood 🤯. - ----- - -******************* -Peek under the hood -******************* - - -.. raw:: html - -
-
- -.. displayitem:: - :header: To Cache or Not to Cache Calls - :description: Learn more about work execution and internal caching. - :col_css: col-md-4 - :button_link: ../../workflows/run_work_once.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Parallel Work - :description: Learn more about work parallelization. - :col_css: col-md-4 - :button_link: ../../workflows/run_work_in_parallel.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Sharing files between works - :description: Learn more about data transfer works. - :col_css: col-md-4 - :button_link: ../../glossary/storage/storage.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Sharing Python Objects between works - :description: Learn more about sharing objects. - :col_css: col-md-4 - :button_link: payload.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Checking Work Status - :description: Learn more about work status. - :col_css: col-md-4 - :button_link: status.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: Handling App Exceptions - :description: Learn more about exception capturing. - :col_css: col-md-4 - :button_link: handling_app_exception.html - :height: 180 - :tag: Advanced - -.. raw:: html - -
-
- -.. raw:: html - -
-
- -.. displayitem:: - :header: Customize your Cloud Compute - :description: Learn more about changing hardware and requirements. - :col_css: col-md-4 - :button_link: compute.html - :height: 180 - :tag: Cloud - -.. raw:: html - -
-
- - ----- - -****************** -Lightning Work API -****************** - -.. raw:: html - -
-
- -.. displayitem:: - :header: LightningWork API - :description: Look into the Lightning API reference. - :col_css: col-md-4 - :button_link: lightning_work.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/core_api/lightning_work/lightning_work.rst b/docs/source-app/core_api/lightning_work/lightning_work.rst deleted file mode 100644 index 54c7328411a3c..0000000000000 --- a/docs/source-app/core_api/lightning_work/lightning_work.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -.. _lightning_work: - -############# -LightningWork -############# - -.. autoclass:: lightning.app.core.work.LightningWork - :exclude-members: _aggregate_status_timeout, _is_state_attribute, _is_state_attribute, set_state - :noindex: diff --git a/docs/source-app/core_api/lightning_work/payload.rst b/docs/source-app/core_api/lightning_work/payload.rst deleted file mode 100644 index cde42af1bd6d2..0000000000000 --- a/docs/source-app/core_api/lightning_work/payload.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -###################################### -Sharing Objects between LightningWorks -###################################### - -**Audience:** Users who want to know how to transfer Python objects between their LightningWorks. - -**Level:** Advanced - -**Prerequisite**: Reach Level 16+, know about the `pandas DataFrames `_ and read and read the :doc:`Access app state guide <../../workflows/access_app_state>`. - ----- - -.. include:: payload_content.rst diff --git a/docs/source-app/core_api/lightning_work/payload_content.rst b/docs/source-app/core_api/lightning_work/payload_content.rst deleted file mode 100644 index 780f3985e30ea..0000000000000 --- a/docs/source-app/core_api/lightning_work/payload_content.rst +++ /dev/null @@ -1,75 +0,0 @@ - -************************************** -What transferring objects does for you -************************************** - -Imagine your application is processing some data using `pandas DaFrame `_ and you want to pass that data to another LightningWork (Work). This is what the **Payload API** is meant for. - ----- - -************************* -Use the Lightning Payload -************************* - -The Payload enables non JSON-serializable attribute objects to be part of your Work's state and to be communicated to other Works. - -Here is an example: - -.. code-block:: python - - import lightning as L - import pandas as pd - - - class SourceWork(L.LightningWork): - def __init__(self): - super().__init__() - self.df = None - - def run(self): - # do some processing - - df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - - # The object you care about needs to be wrapped into a Payload object. - self.df = L.storage.Payload(df) - - # You can access the original object from the payload using its value property. - print("src", self.df.value) - # src col1 col2 - # 0 1 3 - # 1 2 4 - -Once the Payload object is attached to your Work's state, it can be passed to another work using the LightningFlow (Flow) as follows: - -.. code-block:: python - - import lightning as L - import pandas as pd - - - class DestinationWork(L.LightningWork): - def run(self, df: L.storage.Payload): - # You can access the original object from the payload using its value property. - print("dst", df.value) - # dst col1 col2 - # 0 1 3 - # 1 2 4 - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.src = SourceWork() - self.dst = DestinationWork() - - def run(self): - self.src.run() - # The pandas DataFrame created by the ``SourceWork`` - # is passed to the ``DestinationWork``. - # Internally, Lightning pickles and un-pickle the python object, - # so you receive a copy of the original object. - self.dst.run(df=self.src.df) - - - app = L.LightningApp(Flow()) diff --git a/docs/source-app/core_api/lightning_work/status.rst b/docs/source-app/core_api/lightning_work/status.rst deleted file mode 100644 index af3a27ac4047e..0000000000000 --- a/docs/source-app/core_api/lightning_work/status.rst +++ /dev/null @@ -1,13 +0,0 @@ -:orphan: - -#################### -LightningWork Status -#################### - -**Audience:** Users who want to understand ``LightningWork`` under the hood. - -**Level:** Advanced - ----- - -.. include:: status_content.rst diff --git a/docs/source-app/core_api/lightning_work/status_content.rst b/docs/source-app/core_api/lightning_work/status_content.rst deleted file mode 100644 index bb1f2f0273c84..0000000000000 --- a/docs/source-app/core_api/lightning_work/status_content.rst +++ /dev/null @@ -1,197 +0,0 @@ - -************************************* -Everything about LightningWork Status -************************************* - -Statuses indicate transition points in the life of a LightningWork (Work) and contain metadata. - -The different stages are: - -.. code-block:: python - - class WorkStageStatus: - NOT_STARTED = "not_started" - STOPPED = "stopped" - PENDING = "pending" - RUNNING = "running" - SUCCEEDED = "succeeded" - FAILED = "failed" - -And a single status is as follows: - -.. code-block:: python - - @dataclass - class WorkStatus: - stage: WorkStageStatus - timestamp: float - reason: Optional[str] = None - message: Optional[str] = None - count: int = 1 - - -On creation, the Work's status flags all evaluate to ``False`` (in particular ``has_started``) and when calling ``work.run`` in your Lightning Flow (Flow), -the Work transitions from ``is_pending`` to ``is_running`` and then to ``has_succeeded`` if everything went well or ``has_failed`` otherwise. - -.. code-block:: python - - from time import sleep - import lightning as L - - - class Work(L.LightningWork): - def run(self, value: int): - sleep(1) - if value == 0: - return - raise Exception(f"The provided value was {value}") - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.work = Work(raise_exception=False) - self.counter = 0 - - def run(self): - if not self.work.has_started: - print("NOT STARTED") - - elif self.work.is_pending: - print("PENDING") - - elif self.work.is_running: - print("RUNNING") - - elif self.work.has_succeeded: - print("SUCCESS") - - elif self.work.has_failed: - print("FAILED") - - elif self.work.has_stopped: - print("STOPPED") - self.stop() - - print(self.work.status) - self.work.run(self.counter) - self.counter += 1 - - - app = L.LightningApp(Flow()) - -Run this app as follows: - -.. code-block:: bash - - lightning run app test.py > app_log.txt - -And here is the expected output inside ``app_log.txt`` and as expected, -we are observing the following transition ``has_started``, ``is_pending``, ``is_running``, ``has_succeeded``, ``is_running`` and ``has_failed`` - -.. code-block:: console - - NOT STARTED - WorkStatus(stage='not_started', timestamp=1653498225.18468, reason=None, message=None, count=1) - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - PENDING - ... - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - RUNNING - WorkStatus(stage='running', timestamp=1653498228.825194, reason=None, message=None, count=1) - ... - SUCCESS - WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) - SUCCESS - WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) - SUCCESS - WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) - RUNNING - WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) - RUNNING - ... - WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) - RUNNING - WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - ... - -In order to access all statuses: - -.. code-block:: python - - from time import sleep - import lightning as L - - - class Work(L.LightningWork): - def run(self, value: int): - sleep(1) - if value == 0: - return - raise Exception(f"The provided value was {value}") - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.work = Work(raise_exception=False) - self.counter = 0 - - def run(self): - print(self.statuses) - self.work.run(self.counter) - self.counter += 1 - - - app = L.LightningApp(Flow()) - - -Run this app as follows: - -.. code-block:: bash - - lightning run app test.py > app_log.txt - -And here is the expected output inside ``app_log.txt``: - - -.. code-block:: console - - # First execution with value = 0 - - [] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] - ... - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - ... - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] - - # Second execution with value = 1 - - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] - ... - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='failed', timestamp=1653498628.210164, reason='user_exception', message='The provided value was 1', count=1)] - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='failed', timestamp=1653498628.210164, reason='user_exception', message='The provided value was 1', count=1)] diff --git a/docs/source-app/core_api/overview.rst b/docs/source-app/core_api/overview.rst deleted file mode 100644 index 594433acce2aa..0000000000000 --- a/docs/source-app/core_api/overview.rst +++ /dev/null @@ -1,40 +0,0 @@ -:orphan: - -.. _core_api: - -############################### -Learn more about Lightning Core -############################### - -.. raw:: html - -
-
- -.. displayitem:: - :header: Level-up with Lightning Apps - :description: From Basics to Advanced Skills - :col_css: col-md-6 - :button_link: ../levels/basic/index.html - :height: 180 - -.. displayitem:: - :header: Understand Lightning App - :description: Detailed description - :col_css: col-md-6 - :button_link: lightning_app/index.html - :height: 180 - -.. displayitem:: - :header: Understand Lightning Flow - :description: Detailed description - :col_css: col-md-6 - :button_link: lightning_flow.html - :height: 180 - -.. displayitem:: - :header: Understand Lightning Work - :description: Detailed description - :col_css: col-md-6 - :button_link: lightning_work/index.html - :height: 180 diff --git a/docs/source-app/examples/dag/dag.rst b/docs/source-app/examples/dag/dag.rst deleted file mode 100644 index 0df028ebbb0ce..0000000000000 --- a/docs/source-app/examples/dag/dag.rst +++ /dev/null @@ -1,81 +0,0 @@ -:orphan: - -###################################### -Develop a Directed Acyclic Graph (DAG) -###################################### - -.. _dag_example: - -**Audience:** Users coming from MLOps to Lightning Apps, looking for more flexibility. - -A typical ML training workflow can be implemented with a simple DAG. - -Below is a pseudo-code using the lightning framework that uses a LightningFlow to orchestrate the serial workflow: process data, train a model, and serve the model. - -.. code-block:: python - - import lightning as L - - class DAGFlow(L.LightningFlow): - - def __init__(self): - super().__init__() - self.processor = DataProcessorWork(...) - self.train_work = TrainingWork(...) - self.serve_work = ServeWork(...) - - def run(self): - self.processor.run(...) - self.train_work.run(...) - self.serve_work.run(...) - -Below is a pseudo-code to run several works in parallel using a built-in :class:`~lightning.app.structures.Dict`. - -.. code-block:: python - - import lightning as L - - class DAGFlow(L.LightningFlow): - - def __init__(self): - super().__init__() - ... - self.train_works = L.structures.Dict(**{ - "1": TrainingWork(..., parallel=True), - "2": TrainingWork(..., parallel=True), - "3": TrainingWork(..., parallel=True), - ... - }) - ... - - def run(self): - self.processor.run(...) - - # The flow runs through them all, so we need to guard self.serve_work.run - for work in self.train_works.values(): - work.run(...) - - # Wait for all to have finished without errors. - if not all(w.has_succeeded for w in self.train_works): - continue - - self.serve_work.run(...) - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Scheduled DAG with pandas and sklearn from scratch. - :description: DAG example in pure Lightning. - :col_css: col-md-4 - :button_link: dag_from_scratch.html - :height: 180 - :tag: intermediate diff --git a/docs/source-app/examples/dag/dag_from_scratch.rst b/docs/source-app/examples/dag/dag_from_scratch.rst deleted file mode 100644 index 6625317471938..0000000000000 --- a/docs/source-app/examples/dag/dag_from_scratch.rst +++ /dev/null @@ -1,53 +0,0 @@ -:orphan: - -################################################### -Scheduled DAG with pandas and sklearn from scratch. -################################################### - -**Audience:** Users coming from MLOps to Lightning Apps, looking for more flexibility. - -**Level:** Intermediate. - -In this example, you will learn how to create a simple DAG which: - -* Download and process some data -* Train several models and report their associated metrics - -and learn how to schedule this entire process. - -Find the complete example `here `_. - ----- - -************************** -Step 1: Implement your DAG -************************** - -Here is an illustration of the DAG to implement: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/simple_dag.png - :alt: Simple DAG - :width: 100 % - -First, let's define the component we need: - -* DataCollector is responsible to download the data -* Processing is responsible to execute a ``processing.py`` script. -* A collection of model work to train all models in parallel. - -.. literalinclude:: ../../../../examples/app/dag/app.py - :lines: 52-74 - -And its run method executes the steps described above. - -.. literalinclude:: ../../../../examples/app/dag/app.py - :lines: 76-99 - ----- - -***************************** -Step 2: Define the scheduling -***************************** - -.. literalinclude:: ../../../../examples/app/dag/app.py - :lines: 102-130 diff --git a/docs/source-app/examples/data_explore_app.rst b/docs/source-app/examples/data_explore_app.rst deleted file mode 100644 index cd7011a10e93c..0000000000000 --- a/docs/source-app/examples/data_explore_app.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -########################## -Build a Data Exploring App -########################## diff --git a/docs/source-app/examples/etl_app.rst b/docs/source-app/examples/etl_app.rst deleted file mode 100644 index 5b494e943e445..0000000000000 --- a/docs/source-app/examples/etl_app.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -############### -Build a ETL App -############### diff --git a/docs/source-app/examples/file_server/app.py b/docs/source-app/examples/file_server/app.py deleted file mode 100644 index 6c54944fac10d..0000000000000 --- a/docs/source-app/examples/file_server/app.py +++ /dev/null @@ -1,243 +0,0 @@ -import json -import os -import tarfile -import uuid -import zipfile -from pathlib import Path - -from lightning.app import LightningWork, LightningApp -from lightning.app.storage import Drive - - -class FileServer(LightningWork): - def __init__( - self, - drive: Drive, - base_dir: str = "file_server", - chunk_size=10240, - **kwargs - ): - """This component uploads, downloads files to your application. - - Arguments: - drive: The drive can share data inside your application. - base_dir: The local directory where the data will be stored. - chunk_size: The quantity of bytes to download/upload at once. - - """ - super().__init__( - cloud_build_config=BuildConfig(["flask, flask-cors"]), - parallel=True, - **kwargs, - ) - # 1: Attach the arguments to the state. - self.drive = drive - self.base_dir = base_dir - self.chunk_size = chunk_size - - # 2: Create a folder to store the data. - os.makedirs(self.base_dir, exist_ok=True) - - # 3: Keep a reference to the uploaded filenames. - self.uploaded_files = dict() - - def get_filepath(self, path: str) -> str: - """Returns file path stored on the file server.""" - return os.path.join(self.base_dir, path) - - def get_random_filename(self) -> str: - """Returns a random hash for the file name.""" - return uuid.uuid4().hex - - def upload_file(self, file): - """Upload a file while tracking its progress.""" - # 1: Track metadata about the file - filename = file.filename - uploaded_file = self.get_random_filename() - meta_file = uploaded_file + ".meta" - self.uploaded_files[filename] = { - "progress": (0, None), "done": False - } - - # 2: Create a stream and write bytes of - # the file to the disk under `uploaded_file` path. - with open(self.get_filepath(uploaded_file), "wb") as out_file: - content = file.read(self.chunk_size) - while content: - # 2.1 Write the file bytes - size = out_file.write(content) - - # 2.2 Update the progress metadata - self.uploaded_files[filename]["progress"] = ( - self.uploaded_files[filename]["progress"][0] + size, - None, - ) - # 4: Read next chunk of data - content = file.read(self.chunk_size) - - # 3: Update metadata that the file has been uploaded. - full_size = self.uploaded_files[filename]["progress"][0] - self.drive.put(self.get_filepath(uploaded_file)) - self.uploaded_files[filename] = { - "progress": (full_size, full_size), - "done": True, - "uploaded_file": uploaded_file, - } - - # 4: Write down the metadata about the file to the disk - meta = { - "original_path": filename, - "display_name": os.path.splitext(filename)[0], - "size": full_size, - "drive_path": uploaded_file, - } - with open(self.get_filepath(meta_file), "w") as f: - json.dump(meta, f) - - # 5: Put the file to the drive. - # It means other components can access get or list them. - self.drive.put(self.get_filepath(meta_file)) - return meta - - def list_files(self, file_path: str): - # 1: Get the local file path of the file server. - file_path = self.get_filepath(file_path) - - # 2: If the file exists in the drive, transfer it locally. - if not os.path.exists(file_path): - self.drive.get(file_path) - - if os.path.isdir(file_path): - result = set() - for _, _, f in os.walk(file_path): - for file in f: - if not file.endswith(".meta"): - for filename, meta in self.uploaded_files.items(): - if meta["uploaded_file"] == file: - result.add(filename) - return {"asset_names": [v for v in result]} - - # 3: If the filepath is a tar or zip file, list their contents - if zipfile.is_zipfile(file_path): - with zipfile.ZipFile(file_path, "r") as zf: - result = zf.namelist() - elif tarfile.is_tarfile(file_path): - with tarfile.TarFile(file_path, "r") as tf: - result = tf.getnames() - else: - raise ValueError("Cannot open archive file!") - - # 4: Returns the matching files. - return {"asset_names": result} - - def run(self): - # 1: Imports flask requirements. - from flask import Flask, request - from flask_cors import CORS - - # 2: Create a flask app - flask_app = Flask(__name__) - CORS(flask_app) - - # 3: Define the upload file endpoint - @flask_app.post("/upload_file/") - def upload_file(): - """Upload a file directly as form data.""" - f = request.files["file"] - return self.upload_file(f) - - @flask_app.get("/") - def list_files(): - return self.list_files(str(Path(self.base_dir).resolve())) - - # 5: Start the flask app while providing the `host` and `port`. - flask_app.run(host=self.host, port=self.port, load_dotenv=False) - - def alive(self): - """Hack: Returns whether the server is alive.""" - return self.url != "" - - -import requests - -from lightning import LightningWork - - -class TestFileServer(LightningWork): - def __init__(self, drive: Drive): - super().__init__(cache_calls=True) - self.drive = drive - - def run(self, file_server_url: str, first=True): - if first: - with open("test.txt", "w") as f: - f.write("Some text.") - - response = requests.post( - file_server_url + "/upload_file/", - files={'file': open("test.txt", 'rb')} - ) - assert response.status_code == 200 - else: - response = requests.get(file_server_url) - assert response.status_code == 200 - assert response.json() == {"asset_names": ["test.txt"]} - - -from lightning import LightningApp, LightningFlow - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Create a drive to share data between works - self.drive = Drive("lit://file_server") - # 2: Create the filer server - self.file_server = FileServer(self.drive) - # 3: Create the file ser - self.test_file_server = TestFileServer(self.drive) - - def run(self): - # 1: Start the file server. - self.file_server.run() - - # 2: Trigger the test file server work when ready. - if self.file_server.alive(): - # 3 Execute the test file server work. - self.test_file_server.run(self.file_server.url) - self.test_file_server.run(self.file_server.url, first=False) - - # 4 When both execution are successful, exit the app. - if self.test_file_server.num_successes == 2: - self.stop() - - def configure_layout(self): - # Expose the file_server component - # in the UI using its `/` endpoint. - return {"name": "File Server", "content": self.file_server} - - -from lightning.app.runners import MultiProcessRuntime - - -def test_file_server(): - app = LightningApp(Flow()) - MultiProcessRuntime(app).dispatch() - - -from lightning.app.testing import run_app_in_cloud - - -def test_file_server_in_cloud(): - # You need to provide the directory containing the app file. - app_dir = "docs/source-app/examples/file_server" - with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn, name): - """# 1. `admin_page` and `view_page` are playwright Page Objects. - - # Check out https://playwright.dev/python/ doc to learn more. - # You can click the UI and trigger actions. - - # 2. By calling logs = get_logs_fn(), - # you get all the logs currently on the admin page. - - """ diff --git a/docs/source-app/examples/file_server/file_server.rst b/docs/source-app/examples/file_server/file_server.rst deleted file mode 100644 index f9f800a085bb8..0000000000000 --- a/docs/source-app/examples/file_server/file_server.rst +++ /dev/null @@ -1,13 +0,0 @@ -:orphan: - -.. _fileserver_example: - -##################### -Develop a File Server -##################### - -**Prerequisite**: Reach :ref:`level 16+ ` and read the :ref:`Drive article `. - ----- - -.. include:: file_server_content.rst diff --git a/docs/source-app/examples/file_server/file_server_content.rst b/docs/source-app/examples/file_server/file_server_content.rst deleted file mode 100644 index e9e9017232927..0000000000000 --- a/docs/source-app/examples/file_server/file_server_content.rst +++ /dev/null @@ -1,85 +0,0 @@ - - -********* -Our Goal -********* - -Create a simple Lightning App (App) that allows users to upload files and list the uploaded files. - ----- - -************* -Completed App -************* - -Here is a recording of the final App built in this example, tested with pytest. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/file_server.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/file_server.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -********** -App Design -********** - -In order to create this App, we need to develop two components and an App: - -* A **File Server Component** that gives you the ability to download or list files shared with your App. This is particularly useful when you want to trigger an ML job but your users need to provide their own data or if the user wants to download the trained checkpoints. - -* A **Test File Server** Component to interact with the file server. - -* An App putting everything together and the App's associated pytest tests. - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the File Server general structure - :description: Put together the shape of the Component - :col_css: col-md-4 - :button_link: file_server_step_1.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 2: Implement the File Server upload and list files methods - :description: Add the core functionalities to the Component - :col_css: col-md-4 - :button_link: file_server_step_2.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 3: Implement a File Server Testing Component - :description: Create a Component to test the file server - :col_css: col-md-4 - :button_link: file_server_step_3.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 4: Implement tests for the File Server component with pytest - :description: Create an App to validate the upload and list files endpoints - :col_css: col-md-4 - :button_link: file_server_step_4.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/file_server/file_server_step_1.rst b/docs/source-app/examples/file_server/file_server_step_1.rst deleted file mode 100644 index 8703a1d443ef2..0000000000000 --- a/docs/source-app/examples/file_server/file_server_step_1.rst +++ /dev/null @@ -1,49 +0,0 @@ -:orphan: - -################################################## -Step 1: Implement the FileServer general structure -################################################## - -Let’s dive in on how to develop the component with the following code: - -.. literalinclude:: ./app.py - :lines: 1-41, 132-158 - :emphasize-lines: 16, 51- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 2: Implement the File Server upload and list files methods - :description: Add the core functionalities to the Component - :col_css: col-md-4 - :button_link: file_server_step_2.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 3: Implement a File Server Testing Component - :description: Create a Component to test the file server - :col_css: col-md-4 - :button_link: file_server_step_3.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 4: Implement tests for the File Server component with pytest - :description: Create an App to validate the upload and list files endpoints - :col_css: col-md-4 - :button_link: file_server_step_4.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/file_server/file_server_step_2.rst b/docs/source-app/examples/file_server/file_server_step_2.rst deleted file mode 100644 index 86471604a124f..0000000000000 --- a/docs/source-app/examples/file_server/file_server_step_2.rst +++ /dev/null @@ -1,75 +0,0 @@ -:orphan: - -################################################################ -Step 2: Implement the File Server upload and list_files methods -################################################################ - -Let's dive in on how to implement these methods. - -*************************** -Implement the upload method -*************************** - -In this method, we are creating a stream between the uploaded file and the uploaded file stored on the file server disk. - -Once the file is uploaded, we are putting the file into the :class:`~lightning.app.storage.drive.Drive`, so it becomes persistent and accessible to all Components. - -.. literalinclude:: ./app.py - :lines: 12, 51-99 - :emphasize-lines: 49 - -******************************* -Implement the fist_files method -******************************* - -First, in this method, we get the file in the file server filesystem, if available in the Drive. Once done, we list the the files under the provided paths and return the results. - -.. literalinclude:: ./app.py - :lines: 12, 100-130 - :emphasize-lines: 9 - - -******************* -Implement utilities -******************* - -.. literalinclude:: ./app.py - :lines: 12, 43-49 - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the File Server general structure - :description: Put together the shape of the Component - :col_css: col-md-4 - :button_link: file_server_step_1.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 3: Implement a File Server Testing Component - :description: Create a Component to test the file server - :col_css: col-md-4 - :button_link: file_server_step_3.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 4: Implement tests for the File Server component with pytest - :description: Create an App to validate the upload and list files endpoints - :col_css: col-md-4 - :button_link: file_server_step_4.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/file_server/file_server_step_3.rst b/docs/source-app/examples/file_server/file_server_step_3.rst deleted file mode 100644 index 4703ef0750f1e..0000000000000 --- a/docs/source-app/examples/file_server/file_server_step_3.rst +++ /dev/null @@ -1,54 +0,0 @@ -:orphan: - -################################################# -Step 3: Implement a File Server Testing Component -################################################# - -Let's dive in on how to implement a testing component for a server. - -This component needs to test two things: - -* The **/upload_file/** endpoint by creating a file and sending its content to it. - -* The **/** endpoint listing files, by validating the that previously uploaded file is present in the response. - -.. literalinclude:: ./app.py - :lines: 165-182 - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the File Server general structure - :description: Put together the shape of the Component - :col_css: col-md-4 - :button_link: file_server_step_1.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 2: Implement the File Server upload and list files methods - :description: Add the core functionalities to the Component - :col_css: col-md-4 - :button_link: file_server_step_2.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 4: Implement tests for the File Server component with pytest - :description: Create an App to validate the upload and list files endpoints - :col_css: col-md-4 - :button_link: file_server_step_4.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/file_server/file_server_step_4.rst b/docs/source-app/examples/file_server/file_server_step_4.rst deleted file mode 100644 index 04517dfa7baf3..0000000000000 --- a/docs/source-app/examples/file_server/file_server_step_4.rst +++ /dev/null @@ -1,127 +0,0 @@ -:orphan: - -################################################################# -Step 4: Implement tests for the File Server component with pytest -################################################################# - -Let's create a simple App with our **File Server** and **File Server Test** components. - -Once the File Server is up and running, we'll execute the **test_file_server** LightningWork and when both calls are successful, we exit the App using ``self._exit``. - -.. literalinclude:: ./app.py - :lines: 187-218 - - -Simply create a ``test.py`` file with the following code and run ``pytest tests.py``: - -.. literalinclude:: ./app.py - :lines: 221-226 - -To test the App in the cloud, create a ``cloud_test.py`` file with the following code and run ``pytest cloud_test.py``. -Under the hood, we are using the end-to-end testing `playwright `_ library, so you can interact with the UI. - -.. literalinclude:: ./app.py - :lines: 229- - ----- - -******************** -Test the application -******************** - -Clone the Lightning repo and run the following command: - -.. code-block:: bash - - pytest docs/source/examples/file_server/app.py --capture=no -v - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the File Server general structure - :description: Put together the shape of the Component - :col_css: col-md-4 - :button_link: file_server_step_1.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 2: Implement the File Server upload and list files methods - :description: Add the core functionalities to the Component - :col_css: col-md-4 - :button_link: file_server_step_2.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Step 3: Implement a File Server Testing Component - :description: Create a Component to test the file server - :col_css: col-md-4 - :button_link: file_server_step_3.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
- ----- - -****************** -Find more examples -****************** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Develop a DAG - :description: Create a dag pipeline - :col_css: col-md-4 - :button_link: ../dag/dag.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a Github Repo Script Runner - :description: Run any script on github in the cloud - :col_css: col-md-4 - :button_link: ../github_repo_runner/github_repo_runner.html - :height: 150 - :tag: Intermediate - - -.. displayitem:: - :header: Develop a HPO Sweeper - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../hpo/hpo.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a Model Server - :description: Serve multiple models with different parameters - :col_css: col-md-4 - :button_link: ../model_server_app/model_server_app.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/app.py b/docs/source-app/examples/github_repo_runner/app.py deleted file mode 100644 index f57cfcd12dc24..0000000000000 --- a/docs/source-app/examples/github_repo_runner/app.py +++ /dev/null @@ -1,309 +0,0 @@ -import io -import os -import subprocess -import sys -from copy import deepcopy -from functools import partial -from subprocess import Popen -from typing import Dict, List, Optional - -from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow -from lightning.app import structures -from lightning.app.components import TracerPythonScript -from lightning.app.frontend import StreamlitFrontend -from lightning.app.storage.path import Path -from lightning.app.utilities.state import AppState - - -class GithubRepoRunner(TracerPythonScript): - def __init__( - self, - id: str, - github_repo: str, - script_path: str, - script_args: List[str], - requirements: List[str], - cloud_compute: Optional[CloudCompute] = None, - **kwargs, - ): - """The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect - logs. - - Arguments: - id: Identified of the component. - github_repo: The Github Repo URL to clone. - script_path: The path to the script to execute. - script_args: The arguments to be provided to the script. - requirements: The python requirements tp run the script. - cloud_compute: The object to select the cloud instance. - - """ - super().__init__( - script_path=script_path, - script_args=script_args, - cloud_compute=cloud_compute, - cloud_build_config=BuildConfig(requirements=requirements), - **kwargs, - ) - self.id = id - self.github_repo = github_repo - self.logs = [] - - def run(self, *args, **kwargs): - # 1. Hack: Patch stdout so we can capture the logs. - string_io = io.StringIO() - sys.stdout = string_io - - # 2: Use git command line to clone the repo. - repo_name = self.github_repo.split("/")[-1].replace(".git", "") - cwd = os.path.dirname(__file__) - subprocess.Popen( - f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() - - # 3: Execute the parent run method of the TracerPythonScript class. - os.chdir(os.path.join(cwd, repo_name)) - super().run(*args, **kwargs) - - # 4: Get all the collected logs and add them to the state. - # This isn't optimal as heavy, but works for this demo purpose. - self.logs = string_io.getvalue() - string_io.close() - - def configure_layout(self): - return {"name": self.id, "content": self} - - -class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.best_model_path = None - self.best_model_score = None - - def configure_tracer(self): - from lightning.pytorch import Trainer - from lightning.pytorch.callbacks import Callback - - tracer = super().configure_tracer() - - class TensorboardServerLauncher(Callback): - def __init__(self, work): - # The provided `work` is the - # current ``PyTorchLightningScript`` work. - self.w = work - - def on_train_start(self, trainer, *_): - # Add `host` and `port` for tensorboard to work in the cloud. - cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" - server_args = f"--host {self.w.host} --port {self.w.port}" - Popen(cmd + " " + server_args, shell=True) - - def trainer_pre_fn(self, *args, work=None, **kwargs): - # Intercept Trainer __init__ call - # and inject a ``TensorboardServerLauncher`` component. - kwargs["callbacks"].append(TensorboardServerLauncher(work)) - return {}, args, kwargs - - # 5. Patch the `__init__` method of the Trainer - # to inject our callback with a reference to the work. - tracer.add_traced( - Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) - return tracer - - def on_after_run(self, end_script_globals): - import torch - - # 1. Once the script has finished to execute, - # we can collect its globals and access any objects. - trainer = end_script_globals["cli"].trainer - checkpoint_callback = trainer.checkpoint_callback - lightning_module = trainer.lightning_module - - # 2. From the checkpoint_callback, - # we are accessing the best model weights - checkpoint = torch.load(checkpoint_callback.best_model_path) - - # 3. Load the best weights and torchscript the model. - lightning_module.load_state_dict(checkpoint["state_dict"]) - lightning_module.to_torchscript(f"{self.name}.pt") - - # 4. Use lightning.app.storage.Pathto create a reference to the - # torch scripted model. In the cloud with multiple machines, - # by simply passing this reference to another work, - # it triggers automatically a file transfer. - self.best_model_path = Path(f"{self.name}.pt") - - # 5. Keep track of the metrics. - self.best_model_score = float(checkpoint_callback.best_model_score) - - -class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -GITHUB_REPO_RUNNERS = { - "PyTorch Lightning": PyTorchLightningGithubRepoRunner, - "Keras": KerasGithubRepoRunner, - "Tensorflow": TensorflowGithubRepoRunner, -} - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Keep track of the requests within the state - self.requests = [] - # 2: Create a dictionary of components. - self.ws = structures.Dict() - - def run(self): - # Iterate continuously over all requests - for request_id, request in enumerate(self.requests): - self._handle_request(request_id, deepcopy(request)) - - def _handle_request(self, request_id: int, request: Dict): - # 1: Create a name and find selected framework - name = f"w_{request_id}" - ml_framework = request["train"].pop("ml_framework") - - # 2: If the component hasn't been created yet, create it. - if name not in self.ws: - work_cls = GITHUB_REPO_RUNNERS[ml_framework] - work = work_cls(id=request["id"], **request["train"]) - self.ws[name] = work - - # 3: Run the component - self.ws[name].run() - - # 4: Once the component has finished, - # add metadata to the original request for the UI. - if self.ws[name].best_model_path: - request = self.requests[request_id] - request["best_model_score"] = self.ws[name].best_model_score - request["best_model_path"] = self.ws[name].best_model_path - - def configure_layout(self): - # Create a StreamLit UI for the user to run his Github Repo. - return StreamlitFrontend(render_fn=render_fn) - - -def page_1__create_new_run(state): - import streamlit as st - - st.markdown("# Create a new Run 🎈") - - # 1: Collect arguments from the users - id = st.text_input("Name your run", value="my_first_run") - github_repo = st.text_input( - "Enter a Github Repo URL", value="https://github.com/Lightning-AI/lightning-quick-start.git" - ) - - default_script_args = ( - "--trainer.max_epochs=5" - " --trainer.limit_train_batches=4" - " --trainer.limit_val_batches=4" - " --trainer.callbacks=ModelCheckpoint" - " --trainer.callbacks.monitor=val_acc" - ) - default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" - - script_path = st.text_input("Enter your script to run", value="train_script.py") - script_args = st.text_input("Enter your base script arguments", value=default_script_args) - requirements = st.text_input("Enter your requirements", value=default_requirements) - ml_framework = st.radio( - "Select your ML Training Frameworks", options=["PyTorch Lightning", "Keras", "Tensorflow"] - ) - - if ml_framework not in ("PyTorch Lightning"): - st.write(f"{ml_framework} isn't supported yet.") - return - - clicked = st.button("Submit") - - # 2: If clicked, create a new request. - if clicked: - new_request = { - "id": id, - "train": { - "github_repo": github_repo, - "script_path": script_path, - "script_args": script_args.split(" "), - "requirements": requirements.split(" "), - "ml_framework": ml_framework, - }, - } - # 3: IMPORTANT: Add a new request to the state in-place. - # The flow receives the UI request and dynamically create - # and run the associated work from the request information. - state.requests = state.requests + [new_request] - - -def page_2__view_run_lists(state): - import streamlit as st - - st.markdown("# Run Lists 🎈") - # 1: Iterate through all the requests in the state. - for i, r in enumerate(state.requests): - i = str(i) - # 2: Display information such as request, logs, work state, model score. - work = state._state["structures"]["ws"]["works"][f"w_{i}"] - with st.expander(f"Expand to view Run {i}", expanded=False): - if st.checkbox("Expand to view your configuration", key=i): - st.json(r) - if st.checkbox("Expand to view logs", key=i): - st.code(body=work["vars"]["logs"]) - if st.checkbox("Expand to view your work state", key=i): - work["vars"].pop("logs") - st.json(work) - best_model_score = r.get("best_model_score", None) - if best_model_score: - if st.checkbox("Expand to view your run performance", key=i): - st.json({"best_model_score": best_model_score, "best_model_path": r.get("best_model_path")}) - - -def page_3__view_app_state(state): - import streamlit as st - - st.markdown("# App State 🎈") - st.write(state._state) - - -def render_fn(state: AppState): - import streamlit as st - - page_names_to_funcs = { - "Create a new Run": partial(page_1__create_new_run, state=state), - "View your Runs": partial(page_2__view_run_lists, state=state), - "View the App state": partial(page_3__view_app_state, state=state), - } - selected_page = st.sidebar.selectbox( - "Select a page", page_names_to_funcs.keys()) - page_names_to_funcs[selected_page]() - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - # Create the flow - self.flow = Flow() - - def run(self): - # Run the flow - self.flow.run() - - def configure_layout(self): - # 1: Add the main StreamLit UI - selection_tab = [{ - "name": "Run your Github Repo", - "content": self.flow, - }] - # 2: Add a new tab whenever a new work is dynamically created - run_tabs = [e.configure_layout() for e in self.flow.ws.values()] - # 3: Returns the list of tabs. - return selection_tab + run_tabs - - -app = LightningApp(RootFlow()) diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner.rst deleted file mode 100644 index 7e239b2dfd33d..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -.. _github_repo_script_runner_example: - -################################### -Develop a Github Repo Script Runner -################################### - -**Audience:** Users that want to develop interactive applications which runs Github Repo in the cloud at any scale for multiple users. - -**Prerequisite**: Reach :ref:`level 16+ ` and read the docstring of of :class:`~lightning.app.components.python.tracer.TracerPythonScript` component. - ----- - -.. include:: github_repo_runner_content.rst diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst deleted file mode 100644 index 2a0a3aa35c5ac..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst +++ /dev/null @@ -1,97 +0,0 @@ - -******** -Our Goal -******** - -Create a simple Lightning App (App) where users can enter information in a UI to run a given PyTorch Lightning Script from a given Github Repo with some optional extra Python requirements and arguments. - -Users should be able to monitor their training progress in real-time, view the logs, and get the best monitored metric and associated checkpoint for their models. - ----- - -Completed App -^^^^^^^^^^^^^ - -Here is a recording of the final application built in this example. The example is around 200 lines in total and should give you a great foundation to build your own Lightning App. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/github_app.mp4 - :poster: "https://pl-public-data.s3.amazonaws.com/assets_lightning/github_app.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -********** -App Design -********** - -In order to develop the App, we need to build several components: - -* A GithubRepoRunner Component that clones a repo, runs a specific script with provided arguments and collect logs. - -* A PyTorch Lightning GithubRepoRunner Component that augments the GithubRepoRunner component to track PyTorch Lightning Trainer. - -* A UI for the users to provide to trigger dynamically a new execution. - -* A Flow to dynamically create GithubRepoRunner once a user submits information from the UI. - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 2: Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: Step 3: Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 4: Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 5: Put it all together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst deleted file mode 100644 index e85ecc9da6b95..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst +++ /dev/null @@ -1,62 +0,0 @@ -:orphan: - -************************************************ -Step 1: Implement the GithubRepoRunner Component -************************************************ - -The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments, and collect logs. - -Let's dive in on how to develop the component with the following code: - -.. literalinclude:: ./app.py - :lines: -72 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 2: Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: Step 3: Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 4: Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 5: Put it all together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst deleted file mode 100644 index deae8844c8fc2..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -****************************************************************** -Step 2: Implement the PyTorch Lightning GithubRepoRunner Component -****************************************************************** - -The PyTorch Lightning GithubRepoRunner Component subclasses the GithubRepoRunner but tailors the execution experience to PyTorch Lightning. - -As a matter of fact, this component adds two primary tailored features for PyTorch Lightning users: - -* It injects dynamically a custom callback ``TensorboardServerLauncher`` in the PyTorch Lightning Trainer to start a tensorboard server so it can be exposed in Lightning App UI. - -* Once the script has run, the ``on_after_run`` hook of the :class:`~lightning.app.components.python.tracer.TracerPythonScript` is invoked with the script globals, meaning we can collect anything we need. In particular, we are reloading the best model, torch scripting it, and storing its path in the state along side the best metric score. - -Let's dive in on how to develop the component with the following code: - -.. literalinclude:: ./app.py - :lines: 75-136 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 3: Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 4: Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 5: Put it all together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst deleted file mode 100644 index 44cf7dd5a6523..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst +++ /dev/null @@ -1,62 +0,0 @@ -:orphan: - -************************************************** -Step 3: Implement the Flow to manage user requests -************************************************** - -In step 1 and 2, we have implemented the ``GithubRepoRunner`` and ``PyTorchLightningGithubRepoRunner`` components. - -Now, we are going to develop a component to dynamically handle user requests. -Let's dive in on how to develop the component with the following code: - -.. literalinclude:: ./app.py - :lines: 142-190 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 2: Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: Step 4: Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: Step 5: Put it all together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst deleted file mode 100644 index 16893aafee183..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst +++ /dev/null @@ -1,86 +0,0 @@ -:orphan: - -*************************************** -Step 4: Implement the UI with StreamLit -*************************************** - -In step 3, we have implemented a Flow which dynamically creates a Work when a new request is added to the requests list. - -From the UI, we create 3 pages with `StreamLit `_: - -* **Page 1**: Create a form with add a new request to the Flow state **requests**. - -* **Page 2**: Iterate through all the requests and display the associated information. - -* **Page 3**: Display the entire App State. - - -Render All Pages -^^^^^^^^^^^^^^^^ - -.. literalinclude:: ./app.py - :lines: 274-284 - -**Page 1** - -.. literalinclude:: ./app.py - :lines: 193-241 - :emphasize-lines: 43 - -**Page 2** - -.. literalinclude:: ./app.py - :lines: 244-264 - -**Page 3** - -.. literalinclude:: ./app.py - :lines: 267-271 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: 2. Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: 3. Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 5: Put it all together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst deleted file mode 100644 index 9b5b469d5b793..0000000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst +++ /dev/null @@ -1,75 +0,0 @@ -:orphan: - -*************************** -Step 5: Put it all together -*************************** - -Let's dive in on how to develop the component with the following code: - -.. literalinclude:: ./app.py - :lines: 287- - -Run the application -^^^^^^^^^^^^^^^^^^^ - -Clone the Lightning repo and run the following command: - -.. code-block:: bash - - lightning run app docs/source/examples/github_repo_runner/app.py - -Add ``--cloud`` to run this application in the cloud. - -.. code-block:: bash - - lightning run app docs/source/examples/github_repo_runner/app.py --cloud - ----- - -********************** -More hands-on examples -********************** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Develop a DAG - :description: Create a dag pipeline - :col_css: col-md-4 - :button_link: ../dag/dag.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a File Server - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../file_server/file_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a HPO Sweeper - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../hpo/hpo.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a Model Server - :description: Serve multiple models with different parameters - :col_css: col-md-4 - :button_link: ../model_server/model_server.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/hands_on_example.rst b/docs/source-app/examples/hands_on_example.rst deleted file mode 100644 index 57fa1e5ff114a..0000000000000 --- a/docs/source-app/examples/hands_on_example.rst +++ /dev/null @@ -1,50 +0,0 @@ -:orphan: - -################# -Hands-on Examples -################# - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a DAG - :description: Learn how to orchestrate workflows - :col_css: col-md-6 - :button_link: dag/dag.html - :height: 180 - -.. displayitem:: - :header: Build a File Server - :description: Learn how to upload and download files - :col_css: col-md-6 - :button_link: file_server/file_server.html - :height: 180 - -.. displayitem:: - :header: Build a Github Repo Script Runner - :description: Learn how to configure dynamic execution from the UI - :col_css: col-md-6 - :button_link: github_repo_runner/github_repo_runner.html - :height: 180 - -.. displayitem:: - :header: Build a HPO Sweeper - :description: Learn how to scale your training - :col_css: col-md-6 - :button_link: hpo/hpo.html - :height: 180 - -.. displayitem:: - :header: Build a Model Server - :description: Learn how to server your models - :col_css: col-md-6 - :button_link: model_server_app_content.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/index.rst b/docs/source-app/examples/index.rst deleted file mode 100644 index bb7e645da446e..0000000000000 --- a/docs/source-app/examples/index.rst +++ /dev/null @@ -1,36 +0,0 @@ -######## -Examples -######## - -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a DAG workflow - :description: Develop sequential, non-reactive workflows - :col_css: col-md-4 - :button_link: dag/dag.html - :height: 150 - -.. displayitem:: - :header: Develop a File Server - :description: Develop a file server - :col_css: col-md-4 - :button_link: file_server/file_server.html - :height: 150 - -.. displayitem:: - :header: Develop a Github Repo Script Runner - :description: Build an app to run a Github repo - :col_css: col-md-4 - :button_link: github_repo_runner/github_repo_runner.html - :height: 150 - -.. displayitem:: - :header: Deploy a model - :description: Learn how to deploy a model - :col_css: col-md-4 - :button_link: model_server_app/model_server_app.html - :height: 150 diff --git a/docs/source-app/examples/model_server_app/app.py b/docs/source-app/examples/model_server_app/app.py deleted file mode 100644 index 9985014b11912..0000000000000 --- a/docs/source-app/examples/model_server_app/app.py +++ /dev/null @@ -1,34 +0,0 @@ -from locust_component import Locust -from model_server import MLServer -from train import TrainModel - -from lightning import LightningApp, LightningFlow - - -class TrainAndServe(LightningFlow): - def __init__(self): - super().__init__() - self.train_model = TrainModel() - self.model_server = MLServer( - name="mnist-svm", - implementation="mlserver_sklearn.SKLearnModel", - workers=8, - ) - self.performance_tester = Locust(num_users=100) - - def run(self): - self.train_model.run() - self.model_server.run(self.train_model.best_model_path) - if self.model_server.alive(): - # The performance tester needs the model server to be up - # and running to be started, so the URL is added in the UI. - self.performance_tester.run(self.model_server.url) - - def configure_layout(self): - return [ - {"name": "Server", "content": self.model_server.url + "/docs"}, - {"name": "Server Testing", "content": self.performance_tester}, - ] - - -app = LightningApp(TrainAndServe()) diff --git a/docs/source-app/examples/model_server_app/load_testing.rst b/docs/source-app/examples/model_server_app/load_testing.rst deleted file mode 100644 index 9ea347bd0dd44..0000000000000 --- a/docs/source-app/examples/model_server_app/load_testing.rst +++ /dev/null @@ -1,57 +0,0 @@ -:orphan: - -*********************************** -3. Build the Load Testing Component -*********************************** - -Now, we are going to create a component to test the performance of your model server. - -We are going to use a python performance testing tool called `Locust `_. - -.. literalinclude:: ./locust_component.py - - -Finally, once the component is done, we need to create a ``locustfile.py`` file which defines the format of the request to send to your model server. - -The endpoint to hit has the following format: ``/v2/models/{MODEL_NAME}/versions/{VERSION}/infer``. - -.. literalinclude:: ./locustfile.py - - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1. Build a Train Component - :description: Train a model and store its checkpoints with SKlearn - :col_css: col-md-4 - :button_link: train.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 2. Build a Model Server Component - :description: Use MLServer to server your models - :col_css: col-md-4 - :button_link: model_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/locust_component.py b/docs/source-app/examples/model_server_app/locust_component.py deleted file mode 100644 index 432336adf83b3..0000000000000 --- a/docs/source-app/examples/model_server_app/locust_component.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import subprocess - -from lightning import BuildConfig, LightningWork - - -class Locust(LightningWork): - def __init__(self, num_users: int = 100): - """This component checks the performance of a server. The server url is passed to its run method. - - Arguments: - num_users: Number of users emulated by Locust - - """ - # Note: Using the default port 8089 of Locust. - super().__init__( - port=8089, - parallel=True, - cloud_build_config=BuildConfig(requirements=["locust"]), - ) - self.num_users = num_users - - def run(self, load_tested_url: str): - # 1: Create the locust command line. - cmd = " ".join( - [ - "locust", - "--master-host", - str(self.host), - "--master-port", - str(self.port), - "--host", - str(load_tested_url), - "-u", - str(self.num_users), - ] - ) - # 2: Create another process with locust - process = subprocess.Popen(cmd, cwd=os.path.dirname(__file__), shell=True) - - # 3: Wait for the process to finish. As locust is a server, - # this waits infinitely or if killed. - process.wait() diff --git a/docs/source-app/examples/model_server_app/locustfile.py b/docs/source-app/examples/model_server_app/locustfile.py deleted file mode 100644 index 198d6de6cb553..0000000000000 --- a/docs/source-app/examples/model_server_app/locustfile.py +++ /dev/null @@ -1,41 +0,0 @@ -from locust import FastHttpUser, task -from sklearn import datasets -from sklearn.model_selection import train_test_split - - -class HelloWorldUser(FastHttpUser): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._prepare_inference_request() - - @task - def predict(self): - self.client.post( - "/v2/models/mnist-svm/versions/v0.0.1/infer", - json=self.inference_request, - ) - - def _prepare_inference_request(self): - # The digits dataset - digits = datasets.load_digits() - - # To apply a classifier on this data, - # we need to flatten the image, to - # turn the data in a (samples, feature) matrix: - n_samples = len(digits.images) - data = digits.images.reshape((n_samples, -1)) - - # Split data into train and test subsets - _, X_test, _, _ = train_test_split(data, digits.target, test_size=0.5, shuffle=False) - - x_0 = X_test[0:1] - self.inference_request = { - "inputs": [ - { - "name": "predict", - "shape": x_0.shape, - "datatype": "FP32", - "data": x_0.tolist(), - } - ] - } diff --git a/docs/source-app/examples/model_server_app/model_server.py b/docs/source-app/examples/model_server_app/model_server.py deleted file mode 100644 index 10ad770b012cc..0000000000000 --- a/docs/source-app/examples/model_server_app/model_server.py +++ /dev/null @@ -1,90 +0,0 @@ -import json -import subprocess - -from lightning import BuildConfig, LightningWork -from lightning.app.storage.path import Path - -# ML_SERVER_URL = https://github.com/SeldonIO/MLServer - - -class MLServer(LightningWork): - """This components uses SeldonIO MLServer library. - - The model endpoint: /v2/models/{MODEL_NAME}/versions/{VERSION}/infer. - - Arguments: - name: The name of the model for the endpoint. - implementation: The model loader class. - Example: "mlserver_sklearn.SKLearnModel". - Learn more here: $ML_SERVER_URL/tree/master/runtimes - workers: Number of server worker. - - """ - - def __init__( - self, - name: str, - implementation: str, - workers: int = 1, - **kwargs, - ): - super().__init__( - parallel=True, - cloud_build_config=BuildConfig( - requirements=["mlserver", "mlserver-sklearn"], - ), - **kwargs, - ) - # 1: Collect the config's. - self.settings = { - "debug": True, - "parallel_workers": workers, - } - self.model_settings = { - "name": name, - "implementation": implementation, - } - # 2: Keep track of latest version - self.version = 1 - - def run(self, model_path: Path): - """The model is downloaded when the run method is invoked. - - Arguments: - model_path: The path to the trained model. - - """ - # 1: Use the host and port at runtime so it works in the cloud. - # $ML_SERVER_URL/blob/master/mlserver/settings.py#L50 - if self.version == 1: - # TODO: Reload the next version model of the model. - - self.settings.update({"host": self.host, "http_port": self.port}) - - with open("settings.json", "w") as f: - json.dump(self.settings, f) - - # 2. Store the model-settings - # $ML_SERVER_URL/blob/master/mlserver/settings.py#L120 - self.model_settings["parameters"] = { - "version": f"v0.0.{self.version}", - "uri": str(model_path.absolute()), - } - with open("model-settings.json", "w") as f: - json.dump(self.model_settings, f) - - # 3. Launch the Model Server - subprocess.Popen("mlserver start .", shell=True) - - # 4. Increment the version for the next time run is called. - self.version += 1 - - else: - # TODO: Load the next model and unload the previous one. - pass - - def alive(self): - # Current hack, when the url is available, - # the server is up and running. - # This would be cleaned out and automated. - return self.url != "" diff --git a/docs/source-app/examples/model_server_app/model_server.rst b/docs/source-app/examples/model_server_app/model_server.rst deleted file mode 100644 index 283dc97bc99e3..0000000000000 --- a/docs/source-app/examples/model_server_app/model_server.rst +++ /dev/null @@ -1,48 +0,0 @@ -:orphan: - -************************************* -2. Develop the Model Server Component -************************************* - -In the code below, we use `MLServer `_ which aims to provide an easy way to start serving your machine learning models through a REST and gRPC interface, -fully compliant with KFServing's V2 Dataplane spec. - -.. literalinclude:: ./model_server.py - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1. Develop a Train Component - :description: Train a model and store its checkpoints with SKlearn - :col_css: col-md-4 - :button_link: train.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 3. Develop a Load Testing Component - :description: Use Locust to test your model servers - :col_css: col-md-4 - :button_link: load_testing.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the Components together and run the App - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/model_server_app.rst b/docs/source-app/examples/model_server_app/model_server_app.rst deleted file mode 100644 index 933c89d035b00..0000000000000 --- a/docs/source-app/examples/model_server_app/model_server_app.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -.. _model_server_example: - -###################### -Develop a Model Server -###################### - -**Audience:** Users who want to serve their trained models. - -**Prerequisite**: Reach :ref:`level 16+ `. - ----- - -.. include:: model_server_app_content.rst diff --git a/docs/source-app/examples/model_server_app/model_server_app_content.rst b/docs/source-app/examples/model_server_app/model_server_app_content.rst deleted file mode 100644 index 0a9280c53c2b1..0000000000000 --- a/docs/source-app/examples/model_server_app/model_server_app_content.rst +++ /dev/null @@ -1,84 +0,0 @@ - -********* -Objective -********* - -Create a simple application that trains and serves a `Sklearn `_ machine learning model with `MLServer from SeldonIO `_ - ----- - -***************** -Final Application -***************** - -Here is a gif of the final application built in this example. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/ml_server_2.gif - ----- - -************* -System Design -************* - -In order to create such application, we need to build several components: - -* A Model Train Component that trains a model and provides its trained weights - -* A Model Server Component that serves as an API endpoint for the model generated by the **Model Train Component**. - -* A Load Testing Component that tests the model server works as expected. This could be used to CI/CD the performance of newly generated models (left to the users). - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/model_server_app_2.png - -Let's dive into the tutorial. - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1. Build a Train Component - :description: Train a model and store its checkpoints with SKlearn - :col_css: col-md-4 - :button_link: train.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 2. Build a Model Server Component - :description: Use MLServer to server your models - :col_css: col-md-4 - :button_link: model_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 3. Build a Load Testing Component - :description: Use Locust to test your model servers - :col_css: col-md-4 - :button_link: load_testing.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/putting_everything_together.rst b/docs/source-app/examples/model_server_app/putting_everything_together.rst deleted file mode 100644 index 48162a911f1a0..0000000000000 --- a/docs/source-app/examples/model_server_app/putting_everything_together.rst +++ /dev/null @@ -1,80 +0,0 @@ -:orphan: - -****************************** -4. Putting everything together -****************************** - -In the code below, we put together the **TrainWork**, the **MLServer** and the **Locust** components in an ``app.py`` file. - -.. literalinclude:: ./app.py - - -*********** -Run the App -*********** - -To run the app, simply open a terminal and execute this command: - -.. code-block:: bash - - lightning run app docs/source/examples/model_deploy_app/app.py - -Here is a gif of the UI. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/ml_server_2.gif - -.. raw:: html - -
- -Congrats, you have finished the **Build a Model Server** example ! - ----- - -****************** -Find more examples -****************** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Develop a DAG - :description: Develop a DAG pipeline - :col_css: col-md-4 - :button_link: ../dag/dag.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a File Server - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../file_server/file_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a Github Repo Script Runner - :description: Run code from the internet in the cloud - :col_css: col-md-4 - :button_link: ../github_repo_runner/github_repo_runner.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Develop a HPO Sweeper - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../hpo/hpo.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/train.py b/docs/source-app/examples/model_server_app/train.py deleted file mode 100644 index 457cc03cda69d..0000000000000 --- a/docs/source-app/examples/model_server_app/train.py +++ /dev/null @@ -1,41 +0,0 @@ -import joblib -from sklearn import datasets, svm -from sklearn.model_selection import train_test_split - -from lightning import LightningWork -from lightning.app.storage.path import Path - - -class TrainModel(LightningWork): - """This component trains a Sklearn SVC model on digits dataset.""" - - def __init__(self): - super().__init__() - # 1: Add element to the state. - self.best_model_path = None - - def run(self): - # 2: Load the Digits - digits = datasets.load_digits() - - # 3: To apply a classifier on this data, - # we need to flatten the image, to - # turn the data in a (samples, feature) matrix: - n_samples = len(digits.images) - data = digits.images.reshape((n_samples, -1)) - - # 4: Create a classifier: a support vector classifier - classifier = svm.SVC(gamma=0.001) - - # 5: Split data into train and test subsets - X_train, _, y_train, _ = train_test_split(data, digits.target, test_size=0.5, shuffle=False) - - # 6: We learn the digits on the first half of the digits - classifier.fit(X_train, y_train) - - # 7: Save the Sklearn model with `joblib`. - model_file_name = "mnist-svm.joblib" - joblib.dump(classifier, model_file_name) - - # 8: Keep a reference the the generated model. - self.best_model_path = Path("mnist-svm.joblib") diff --git a/docs/source-app/examples/model_server_app/train.rst b/docs/source-app/examples/model_server_app/train.rst deleted file mode 100644 index fdb6f6a93a0a7..0000000000000 --- a/docs/source-app/examples/model_server_app/train.rst +++ /dev/null @@ -1,49 +0,0 @@ -:orphan: - -**************************** -1. Build the Train Component -**************************** - -In the code below, we create a work which trains a simple `SVC `_ model on the digits dataset (classification). - -Once the model is trained, it is saved and a reference :class:`~lightning.app.storage.path.Path` with ``best_model_path`` state attribute. - -.. literalinclude:: ./train.py - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 2. Build a Model Server Component - :description: Use MLServer to server your models - :col_css: col-md-4 - :button_link: model_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 3. Build a Load Testing Component - :description: Use Locust to test your model servers - :col_css: col-md-4 - :button_link: load_testing.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/research_demo_app.rst b/docs/source-app/examples/research_demo_app.rst deleted file mode 100644 index 90276f9b95f97..0000000000000 --- a/docs/source-app/examples/research_demo_app.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -######################### -Build a Research Demo App -######################### diff --git a/docs/source-app/get_started/add_an_interactive_demo.rst b/docs/source-app/get_started/add_an_interactive_demo.rst deleted file mode 100644 index 0ad0e6b9c8771..0000000000000 --- a/docs/source-app/get_started/add_an_interactive_demo.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -####################### -Add an Interactive Demo -####################### - -.. _add_an_interactive_Demo: - -**Required background:** Basic Python familiarity and complete the install guide. - -**Goal:** We'll walk you through the 4 key steps to run a Lightning App that trains and demos a model. - ----- - -.. include:: go_beyond_training_content.rst diff --git a/docs/source-app/get_started/build_model.rst b/docs/source-app/get_started/build_model.rst deleted file mode 100644 index 300b220ee61ce..0000000000000 --- a/docs/source-app/get_started/build_model.rst +++ /dev/null @@ -1,73 +0,0 @@ -:orphan: - -.. _build_model: - -####################### -Build and Train a Model -####################### - -**Required background:** Basic Python familiarity and complete the guide. - -**Goal:** We'll walk you through the creation of a model using PyTorch Lightning. - ----- - -********************************* -A simple PyTorch Lightning script -********************************* - -Let's assume you already have a folder with those two files. - -.. code-block:: bash - - pl_project/ - train.py # your own script to train your models - requirements.txt # your python requirements. - -If you don't, simply create a ``pl_project`` folder with those two files and add the following `PyTorch Lightning `_ code in the ``train.py`` file. This code trains a simple ``AutoEncoder`` on `MNIST Dataset `_. - -.. literalinclude:: ../code_samples/convert_pl_to_app/train.py - -Add the following to the ``requirements.txt`` file. - -.. literalinclude:: ../code_samples/convert_pl_to_app/requirements.txt - -Simply run the following commands in your terminal to install the requirements and train the model. - -.. code-block:: bash - - pip install -r requirements.txt - python train.py - -Get through `PyTorch Lightning Introduction `_ to learn more. - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
-
- -.. displayitem:: - :header: Evolve a Model into an ML System - :description: Develop an App to train a model in the cloud - :col_css: col-md-6 - :button_link: training_with_apps.html - :height: 180 - -.. displayitem:: - :header: Start from a Template ML System - :description: Learn about Apps, from a template. - :col_css: col-md-6 - :button_link: go_beyond_training.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/get_started/go_beyond_training.rst b/docs/source-app/get_started/go_beyond_training.rst deleted file mode 100644 index f45e7f9f0ab62..0000000000000 --- a/docs/source-app/get_started/go_beyond_training.rst +++ /dev/null @@ -1,14 +0,0 @@ -:orphan: - -################################ -Start from an ML system template -################################ - -.. _go_beyond_training: - -**Required background:** Basic Python familiarity and complete the install guide. - -**Goal:** We'll walk you through the 4 key steps to run a Lightning App that trains and demos a model. - - -.. include:: go_beyond_training_content.rst diff --git a/docs/source-app/get_started/go_beyond_training_content.rst b/docs/source-app/get_started/go_beyond_training_content.rst deleted file mode 100644 index 5bc632b08ddd0..0000000000000 --- a/docs/source-app/get_started/go_beyond_training_content.rst +++ /dev/null @@ -1,405 +0,0 @@ -************************************************ -The *Train & Demo PyTorch Lightning* Application -************************************************ - -Find the *Train & Demo PyTorch Lightning* application in the `Lightning.ai App Gallery `_. - -Here is a recording of this App running locally and in the cloud with the same behavior. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_app_experience_cut.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_app_experience_cut.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -In the steps below, we are going to show you how to build this application. - -Here are `the entire App's code `_ and `its commented components. `_ - ----- - -************************* -Step 1: Install Lightning -************************* - -If you are using a virtual env, don't forget to activate it before running commands. -You must do so in every new shell. - -.. tip:: We highly recommend using virtual environments. - -.. code:: bash - - pip install lightning - ----- - -**************************************** -Step 2: Install the *Train and Demo* App -**************************************** -The first Lightning App we'll explore is an App to train and demo a machine learning model. - -.. - [|qs_code|], [|qs_live_app|]. - - .. |qs_live_app| raw:: html - - live app - - .. |qs_code| raw:: html - - code - - -Install this App by typing: - -.. code-block:: bash - - lightning_app install app lightning/quick-start - -Verify the App was successfully installed: - -.. code-block:: bash - - cd lightning-quick-start - ----- - -*************************** -Step 3: Run the App locally -*************************** - -Run the app locally with the ``run`` command 🤯 - -.. code:: bash - - lightning_app run app app.py - ----- - -******************************** -Step 4: Run the App in the cloud -******************************** - -Add the ``--cloud`` argument to run on the `Lightning.AI cloud `_. 🤯🤯🤯 - -.. code:: bash - - lightning_app run app app.py --cloud - -.. - Your app should look like this one (|qs_live_app|) - ----- - -******************* -Understand the code -******************* -The App that we just launched trained a PyTorch Lightning model (although any framework works), then added an interactive demo. - -This is the App's code: - -.. code:: python - - # lightning-quick-start/app.py - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute()) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - app = L.LightningApp(TrainDeploy()) - -Let's break down the code section by section to understand what it is doing. - ----- - -1: Define root component -^^^^^^^^^^^^^^^^^^^^^^^^ - -A Lightning App provides a cohesive product experience for a set of unrelated components. - -The top-level component (Root) must subclass ``L.LightningFlow`` - - -.. code:: python - :emphasize-lines: 6 - - # lightning-quick-start/app.py - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - app = L.LightningApp(TrainDeploy()) - ----- - -2: Define components -^^^^^^^^^^^^^^^^^^^^ -In the __init__ method, we define the components that make up the App. In this case, we have 2 components, -a component to execute any PyTorch Lightning script (model training) and a second component to -start a Gradio server for demo purposes. - -.. code:: python - :emphasize-lines: 9, 14 - - # lightning-quick-start/app.py - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - app = L.LightningApp(TrainDeploy()) - ----- - -3: Define how components Flow -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Every component has a ``run`` method. The run method defines the 🌊 Flow 🌊 of how components interact together. - -In this case, we train a model (until completion). When it's done AND there exists a checkpoint, we launch a -demo server: - -.. code:: python - :emphasize-lines: 18, 21, 22 - - # lightning-quick-start/app.py - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - app = L.LightningApp(TrainDeploy()) - -.. note:: If you've used other ML systems you'll be pleasantly surprised to not find decorators or YAML files. - ----- - -4: Connect web user interfaces -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -All our favorite tools normally have their own web user interfaces (UI). - -Implement the ``configure_layout`` method to connect them together: - -.. code:: python - :emphasize-lines: 24-27 - - # lightning-quick-start/app.py - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - app = L.LightningApp(TrainDeploy()) - ----- - -5: Init the ``app`` object -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Initialize an ``app`` object with the ``TrainDeploy`` component (this won't run the App yet): - -.. code:: python - :emphasize-lines: 29 - - # lightning-quick-start/app.py - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - app = L.LightningApp(TrainDeploy()) - ----- - -****************************** -What components are supported? -****************************** -Any component can work with Lightning AI! - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/Lightning.gif - :alt: What is Lightning gif. - :width: 100 % - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add components to your App - :description: Expand your App by adding components. - :col_css: col-md-4 - :button_link: ../workflows/extend_app.html - :height: 180 - -.. displayitem:: - :header: Build a component - :description: Learn to build your own component. - :col_css: col-md-4 - :button_link: ../workflows/build_lightning_component/index.html - :height: 180 - -.. displayitem:: - :header: Explore more Apps - :description: Explore more apps for inspiration. - :col_css: col-md-4 - :button_link: https://lightning.ai/apps - :height: 180 - -.. displayitem:: - :header: Under the hood - :description: Explore how it works under the hood. - :col_css: col-md-4 - :button_link: ../core_api/lightning_app/index.html - :height: 180 - -.. displayitem:: - :header: Run on your private cloud - :description: Run Lightning Apps on your private VPC or on-prem. - :button_link: ../workflows/run_on_private_cloud.html - :col_css: col-md-4 - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/get_started/jumpstart_from_app_gallery.rst b/docs/source-app/get_started/jumpstart_from_app_gallery.rst deleted file mode 100644 index c6a49674c9aa9..0000000000000 --- a/docs/source-app/get_started/jumpstart_from_app_gallery.rst +++ /dev/null @@ -1,123 +0,0 @@ -:orphan: - -##################################### -Start from Ready-to-Run Template Apps -##################################### - -.. _jumpstart_from_app_gallery: - -Anyone can build Apps for their own use cases and promote them on the `App Gallery `_. - -In return, you can benefit from the work of others and get started faster by re-using a ready-to-run App close to your own use case. - - -************* -User Workflow -************* - -#. Visit the `App Gallery `_ and look for an App close to your own use case. - - .. raw:: html - -
- -#. If **Launch** is available, it means the App is live and ready to be used! Take it for a spin. - - .. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/launch_button.png - :alt: Launch Button on lightning.ai - :width: 100 % - -#. By clicking **Clone & Run**, a copy of the App is added to your account and an instance starts running. - - - .. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/clone_and_run.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/clone_and_run.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -#. If you found an App that matches what you need, move to **step 5**! Otherwise, go back to **step 1**. - - .. raw:: html - -
- -#. Copy the installation command (optionally from the clipboard on the right). - - .. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/install_command.png - :alt: Install command on lightning.ai - :width: 100 % - -#. Copy the command to your local terminal. - - .. code-block:: bash - - lightning_app install app lightning/hackernews-app - -#. Go through the installation steps. - - .. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/install_an_app.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/install_an_app.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -#. Run the App locally. - - .. code-block:: bash - - cd LAI-Hackernews-App - lightning_app run app app.py - - .. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/hackernews.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/hackernews.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -#. Open the code with your favorite IDE, modify it, and run it back in the cloud. - - .. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/hackernews_modified.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/hackernews_modified.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add Component made by others to your App - :description: Add more functionality to your projects - :col_css: col-md-6 - :button_link: jumpstart_from_component_gallery.html - :height: 180 - -.. displayitem:: - :header: Level-up your skills with Lightning Apps - :description: From Basic to Advanced Skills - :col_css: col-md-6 - :button_link: ../levels/basic/index.html - :height: 180 - -.. raw:: html - -
-
-
diff --git a/docs/source-app/get_started/jumpstart_from_component_gallery.rst b/docs/source-app/get_started/jumpstart_from_component_gallery.rst deleted file mode 100644 index 95f7d570d87d0..0000000000000 --- a/docs/source-app/get_started/jumpstart_from_component_gallery.rst +++ /dev/null @@ -1,151 +0,0 @@ -:orphan: - -######################################## -Add Component made by others to your App -######################################## - -.. _jumpstart_from_component_gallery: - -Anyone can build components for their own use case and promote them on the `Component Gallery `_. - -In return, you can benefit from the work of others and add new functionalities to your Apps with minimal effort. - - -************* -User Workflow -************* - -#. Visit the `Component Gallery `_ and look for a Component close to something you want to do. - - .. raw:: html - -
- -#. Check out the code for inspiration or simply install the component from PyPi and use it. - ----- - -************* -Success Story -************* - -The default `Train and Demo Application `_ trains a PyTorch Lightning -model and then starts a demo with `Gradio `_. - -.. code-block:: python - - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - - app = L.LightningApp(TrainDeploy()) - -However, someone who wants to use this Aop (maybe you) found `Lightning HPO `_ -from browsing the `Component Gallery `_ and decided to give it a spin after checking the associated -`Github Repository `_. - -Once ``lightning_hpo`` installed, they improved the default App by easily adding HPO support to their project. - -Here is the resulting App. It is almost the same code, but it's way more powerful now! - -This is the power of `lightning.ai `_ ecosystem 🔥⚡🔥 - -.. code-block:: python - - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - import optuna - from optuna.distributions import LogUniformDistribution - from lightning_hpo import Optimizer, BaseObjective - - - class HPOPyTorchLightningScript(PyTorchLightningScript, BaseObjective): - @staticmethod - def distributions(): - return {"model.lr": LogUniformDistribution(0.0001, 0.1)} - - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = Optimizer( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - objective_cls=HPOPyTorchLightningScript, - n_trials=4, - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work.hi_plot} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - - app = L.LightningApp(TrainDeploy()) - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Start from Ready-to-Run Template Apps - :description: Jump-start your projects development - :col_css: col-md-6 - :button_link: jumpstart_from_app_gallery.html - :height: 180 - -.. displayitem:: - :header: Level-up your skills with Lightning Apps - :description: From Basic to Advanced Skills - :col_css: col-md-6 - :button_link: ../levels/basic/index.html - :height: 180 - -.. raw:: html - -
-
-
diff --git a/docs/source-app/get_started/training_with_apps.rst b/docs/source-app/get_started/training_with_apps.rst deleted file mode 100644 index 46cc851bdd290..0000000000000 --- a/docs/source-app/get_started/training_with_apps.rst +++ /dev/null @@ -1,125 +0,0 @@ -:orphan: - -################################ -Evolve a model into an ML system -################################ - -.. _convert_pl_to_app: - -**Required background:** Basic Python familiarity and complete the :ref:`build_model` guide. - -**Goal:** We'll walk you through the two key steps to build your first Lightning App from your existing PyTorch Lightning scripts. - - -******************* -Training and beyond -******************* - -With `PyTorch Lightning `__, we abstracted distributed training and hardware, by organizing PyTorch code. -With `Lightning Apps `__, we unified the local and cloud experience while abstracting infrastructure. - -By using `PyTorch Lightning `__ and `Lightning Apps `__ -together, a completely new world of possibilities emerges. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/pl_to_app_4.png - :alt: From PyTorch Lightning to Lightning App - :width: 100 % - ----- - -****************************************** -1. Write an App to run the train.py script -****************************************** - -This article continues where the :ref:`build_model` guide finished. - -Create an additional file ``app.py`` in the ``pl_project`` folder as follows: - -.. code-block:: bash - - pl_project/ - app.py - train.py - requirements.txt - -Inside the ``app.py`` file, add the following code. - -.. literalinclude:: ../code_samples/convert_pl_to_app/app.py - -This App runs the PyTorch Lightning script contained in the ``train.py`` file using the powerful :class:`~lightning.app.components.python.tracer.TracerPythonScript` component. This is really worth checking out! - ----- - -************************************************ -2. Run the train.py file locally or in the cloud -************************************************ - -First, go to the ``pl_folder`` folder from the local terminal and install the requirements. - -.. code-block:: bash - - cd pl_folder - pip install -r requirements.txt - -To run your app, copy the following command to your local terminal: - -.. code-block:: bash - - lightning_app run app app.py - -Simply add ``--cloud`` to run this application in the cloud with a GPU machine 🤯 - -.. code-block:: bash - - lightning_app run app app.py --cloud - - -Congratulations! Now, you know how to run a `PyTorch Lightning `_ script with Lightning Apps. - -Lightning Apps can make your ML system way more powerful, keep reading to learn how. - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Level-up with Lightning Apps - :description: From Basics to Advanced Skills - :col_css: col-md-4 - :button_link: ../levels/basic/index.html - :height: 180 - -.. displayitem:: - :header: Add an Interactive Demo - :description: Add a Gradio Demo once the training is finished - :col_css: col-md-4 - :button_link: add_an_interactive_demo.html - :height: 180 - -.. displayitem:: - :header: Add Model Serving - :description: Serve and load testing with MLServer and Locust - :col_css: col-md-4 - :button_link: ../examples/model_server_app/model_server_app.html - :height: 180 - -.. displayitem:: - :header: Add DAG Orchestration - :description: Organize your processing, training and metrics collection - :col_css: col-md-4 - :button_link: ../examples/dag/dag.html - :height: 180 - -.. displayitem:: - :header: Add Team Collaboration - :description: Create an app to run any PyTorch Lightning Script from Github - :col_css: col-md-4 - :button_link: ../examples/github_repo_runner/github_repo_runner.html - :height: 180 diff --git a/docs/source-app/get_started/what_app_can_do.rst b/docs/source-app/get_started/what_app_can_do.rst deleted file mode 100644 index b4033dd40d594..0000000000000 --- a/docs/source-app/get_started/what_app_can_do.rst +++ /dev/null @@ -1,187 +0,0 @@ -:orphan: - -############################################ -Discover what Lightning Apps can do in 5 min -############################################ - -.. _what_app_can_do: - -Lightning Apps can be plenty things, and while a picture is worth a thousand words, videos showing you examples should be worth even more. - - -***************************** -Flashy - Auto ML App (Public) -***************************** - -Train a model on any image or text dataset without writing any code. Flashy uses `React.js `_ for its frontend. - -Find `Flashy `_ on the App Gallery and the `Flashy codebase. `_ on GitHub. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/flashy.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/flashy.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -.. ---- - -.. *************************************** -.. NVIDIA Omniverse Sampling App (Private) -.. *************************************** - -.. Use `Nvidia Sampling Omniverse `_ to generate synthetic samples from 3D meshes and train an object detector on that data. - -.. .. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/Omniverse-Sampling.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/Omniverse-Sampling.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -********************* -Research App (Public) -********************* - -Share your paper ``bundled`` with the arxiv link, poster, live jupyter notebook, interactive demo to try the model, and more! - -Find the `Research App `_ on the App Gallery and the `Research App codebase. `_ on GitHub. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/research_app.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/research_app.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -************************************************ -ScratchPad - Notebook Manager for Team (Public) -************************************************ - -Run multiple Jupyter Notebooks on cloud CPUs or machines with multiple GPUs. - -Find the `ScratchPad App `_ on the App Gallery and the `ScratchPad App codebase `_ on GitHub. - -.. note:: ScratchPad is `tested end-to-end `_ on every Lightning App commit with `pytest `_. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/notebook_apps.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/notebook_apps.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -*********************** -InVideo Search (Public) -*********************** - -This App lets you find anything you're looking for inside a video. The engine is powered by `Open AI CLIP `_. - -Find the `InVideo Search App `_ on the App Gallery and the `InVideo Search App codebase. `_ in GitHub. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/video_search_2.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/video_search_2.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -****************************** -AI-powered HackerNews (Public) -****************************** - -Save yourself time, and get Hacker News story recommendations, chosen for you specifically. This Lightning App was designed to illustrate a full end-to-end MLOPs workflow aimed at enterprise recommendation systems. - -Find the `AI-powered HackerNews App `_ on the App Gallery and the `AI-powered HackerNews App codebase. `_ on GitHub. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/hackernews_app.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/hackernews_app.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - ----- - -********************************************************************* -Lightning Apps can turn ML into scalable systems in days — not months -********************************************************************* - -Use the Lightning framework to develop any ML system: train and deploy a model, create an ETL pipeline, -or spin up a research demo — using the intuitive principles we pioneered with PyTorch Lightning. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/apps_logos_2.png - :alt: Apps with Logos - :width: 100 % - -Anyone who knows Python can build a Lightning App, even without machine learning experience. - -Lightning Apps are: - -- cloud agnostic -- fault-tolerant, distributed, cost optimized -- production ready -- local and cloud debuggable -- highly reactive & interactive -- connect multiple UIs together -- built for team collaboration -- framework agnostic, use your own stack -- and much more - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_app_experience_cut.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_app_experience_cut.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -********** -Next Steps -********** - -.. raw:: html - -
-
-
- -.. displayitem:: - :header: Build & Train a Model - :description: Discover PyTorch Lightning and train your first Model. - :col_css: col-md-4 - :button_link: build_model.html - :height: 180 - -.. displayitem:: - :header: Evolve a Model into an ML System - :description: Develop an App to train a model in the cloud - :col_css: col-md-4 - :button_link: training_with_apps.html - :height: 180 - -.. displayitem:: - :header: Start from an ML system template - :description: Learn about Apps, from a template. - :col_css: col-md-4 - :button_link: go_beyond_training.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/app_tree.rst b/docs/source-app/glossary/app_tree.rst deleted file mode 100644 index 37413f30cfe0f..0000000000000 --- a/docs/source-app/glossary/app_tree.rst +++ /dev/null @@ -1,113 +0,0 @@ -:orphan: - -.. _app_component_tree: - -################### -App Component Tree -################### - -**Audience:** Users who want to know how components can be composed with each other. - -**Level:** Basic - ----- - -************************************** -What is an Application Component Tree? -************************************** - -Components can be nested to form component trees where the LightningFlows are its branches and LightningWorks are its leaves. - -This design enables users to organize and maintain their code with more ease, but more importantly, this helps creating an ecosystem with reusable components. - -Here's a basic application with four flows and two works (associated tree structure): - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/tree.gif - :alt: Basic App Components - :width: 100 % - - -.. literalinclude:: ../code_samples/quickstart/app_comp.py - -A Lightning app runs all flows into a single process. Its flows coordinate the execution of the works each running in their own independent processes. - ----- - -*********************************************** -How do I define my application component tree? -*********************************************** - -In order to define your application component tree, you need create a tree of components and attach them to your root flow. - -You can attach your components in the **__init__** method of a flow. - -.. code-block:: python - - import lightning as L - - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - # The `Work` component is attached here. - self.work = Work() - - # The `NestedFlow` component is attached here. - self.nested_flow = NestedFlow() - -Once done, simply add the root flow to a Lightning app as follows: - -.. code-block:: python - - app = L.LightningApp(RootFlow()) - ----- - -****************************************** -Is my application component tree static? -****************************************** - -No, Lightning supports dynamic flows and works. - -You can simply attach your components in the **run** method of a flow using the Python functions **hasattr**, **setattr**, and **getattr**. - -.. code-block:: python - - class RootFlow(L.LightningFlow): - def run(self): - - if not hasattr(self, "work"): - # The `Work` component is attached here. - setattr(self, "work", Work()) - # Run the `Work` component. - getattr(self, "work").run() - - if not hasattr(self, "nested_flow"): - # The `NestedFlow` component is attached here. - setattr(self, "nested_flow", NestedFlow()) - # Run the `NestedFlow` component. - getattr(self, "wonested_flowrk").run() - - -But it is usually more readable to use Lightning built-in :class:`~lightning.app.structures.Dict` or :class:`~lightning.app.structures.List` as follows: - -.. code-block:: python - - from lightning.app.structures import Dict - - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.dict = Dict() - - def run(self): - if "work" not in self.dict: - # The `Work` component is attached here. - self.dict["work"] = Work() - self.dict["work"].run() - - if "nested_flow" not in self.dict: - # The `NestedFlow` component is attached here. - self.dict["nested_flow"] = NestedFlow() - self.dict["nested_flow"].run() diff --git a/docs/source-app/glossary/build_config/build_config.rst b/docs/source-app/glossary/build_config/build_config.rst deleted file mode 100644 index 43ba0b02e2bcf..0000000000000 --- a/docs/source-app/glossary/build_config/build_config.rst +++ /dev/null @@ -1,43 +0,0 @@ -:orphan: - -.. _build_config: - -################### -Build Configuration -################### - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Basic - :description: Learn how to manage Python dependencies for an individual LightningWork - :col_css: col-md-6 - :button_link: build_config_basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Intermediate - :description: Learn how to run custom build commands for a LightningWork - :col_css: col-md-6 - :button_link: build_config_intermediate.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Advanced - :description: Learn how to use a custom Docker image for a LightningWork - :col_css: col-md-6 - :button_link: build_config_advanced.html - :height: 150 - :tag: advanced - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/build_config/build_config_advanced.rst b/docs/source-app/glossary/build_config/build_config_advanced.rst deleted file mode 100644 index bc6f5a2062d12..0000000000000 --- a/docs/source-app/glossary/build_config/build_config_advanced.rst +++ /dev/null @@ -1,63 +0,0 @@ -:orphan: - -############################## -Build Configuration (Advanced) -############################## - -**Audience:** Users who want full control over the docker image that is being installed in the cloud. - -**Level:** Advanced - -Advanced users who need full control over the environment a LightningWork runs in can specify a custom docker image that will be deployed in the cloud. - - ----- - -****************** -Use a docker image -****************** - -Create a :class:`~lightning.app.utilities.packaging.build_config.BuildConfig` and provide a **publicly accessible** link to where the image is hosted: - -.. code-block:: python - - from lightning.app import LightningWork, BuildConfig - - - class MyWork(LightningWork): - def __init__(self): - super().__init__() - - # Using a publicly hosted docker image: - self.cloud_build_config = BuildConfig( - # This is one of the base images Lightning uses by default - image="ghcr.io/gridai/base-images:v1.8-gpu" - ) - - # Can also be combined with extra requirements - self.cloud_build_config = BuildConfig(image="...", requirements=["torchmetrics"]) - - -.. warning:: - Many public hosters like DockerHub apply rate limits for public images. We recommend to pull images from your own registry. - For example, you can set up a - `docker registry on GitHub `_. - - -.. note:: - - The build config only applies when running in the cloud and gets ignored otherwise. A local build config is currently not supported. - - Images from private registries are currently not supported. - -.. note:: - Custom docker images must have python installed. We'll use `virtualenv` from this system python to create a virtual environment. - We'll also configure the `virtualenv` to use the packages installed under system's python so your packages are not lost - ----- - - -********************* -Provide a docker file -********************* - -.. note:: - Not yet supported. Coming soon. diff --git a/docs/source-app/glossary/build_config/build_config_basic.rst b/docs/source-app/glossary/build_config/build_config_basic.rst deleted file mode 100644 index 538cf868c5642..0000000000000 --- a/docs/source-app/glossary/build_config/build_config_basic.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -########################### -Build Configuration (Basic) -########################### - -**Audience:** Users who need to install Python packages for an individual LightningWork. - -**Level:** Basic - ----- - -*********************************** -List dependencies in separate files -*********************************** - -If you are building an app with multiple LightningWorks that have different or even conflicting requirements, split your dependencies into individual files -for more granular control. - -.. code-block:: bash - - ├── app.py - ├── requirements.txt # Global requirements for the entire app - └── works - ├── serve - │ ├── requirements.txt # Requirements specific to the 'serve' work - │ └── serve.py # Source file for the LightningWork - └── train - ├── requirements.txt # Requirements specific to the 'train' work - └── train.py # Source file for the LightningWork - -The requirements.txt file must be located in the same directory as the source file of the LightningWork. -When the LightningWork starts up, it will pick up the requirements file if present and install all listed packages. - -.. note:: - This only applies when running in the cloud. The requirements.txt files get ignored when running locally. - ----- - -*********************************** -Define the requirements in the code -*********************************** - -Instead of listing the requirements in a file, you can also pass them to the LightningWork at runtime using the -:class:`~lightning.app.utilities.packaging.build_config.BuildConfig`: - -.. code-block:: python - :emphasize-lines: 7 - - from lightning.app import LightningWork, BuildConfig - - - class MyWork(LightningWork): - def __init__(self): - super().__init__() - self.cloud_build_config = BuildConfig(requirements=["torch>=1.8", "torchmetrics"]) - -.. note:: - The build config only applies when running in the cloud and gets ignored otherwise. A local build config is currently not supported. - -.. warning:: - Custom base images are not supported with the default CPU cloud compute. For example: - - .. code-block:: py - - class MyWork(LightningWork): - def __init__(self): - super().__init__(cloud_build_config=BuildConfig(image="my-custom-image")) # no cloud compute, for example default work diff --git a/docs/source-app/glossary/build_config/build_config_intermediate.rst b/docs/source-app/glossary/build_config/build_config_intermediate.rst deleted file mode 100644 index 600b8e85f36ef..0000000000000 --- a/docs/source-app/glossary/build_config/build_config_intermediate.rst +++ /dev/null @@ -1,56 +0,0 @@ -:orphan: - -################################## -Build Configuration (Intermediate) -################################## - -**Audience:** Users who need to execute commands to configure the machine before a LightningWork runs on it. - -**Level:** Intermediate - -When a LightningWork machine starts up in the cloud, it uses a lightweight operating system with essential packages pre-installed. -If you need to install additional system packages or run other configuration steps before your code executes on that machine, it is possible to do so by creating a custom -:class:`~lightning.app.utilities.packaging.build_config.BuildConfig`: - -1. Subclass :class:`~lightning.app.utilities.packaging.build_config.BuildConfig`: - - .. code-block:: python - - from lightning.app import BuildConfig - - - @dataclass - class CustomBuildConfig(BuildConfig): - def build_commands(self): - return ["sudo apt-get install libsparsehash-dev"] - - -2. Set the build config on the LightningWork: - - .. code-block:: python - - from lightning.app import LightningWork - - - class MyWork(LightningWork): - def __init__(self): - super().__init__() - - # Use the custom build config - self.cloud_build_config = CustomBuildConfig() - - # Can also be combined with extra requirements - self.cloud_build_config = CustomBuildConfig(requirements=["torchmetrics"]) - -.. note:: - - When you need to execute commands or install tools that require more privileges than the current user has, you can use ``sudo`` without needing to provide a password, e.g., when installing system packages. - - The build config only applies when running in the cloud and gets ignored otherwise. A local build config is currently not supported. - -.. warning:: - Custom base images are not supported with the default CPU cloud compute. For example: - - .. code-block:: py - - class MyWork(LightningWork): - def __init__(self): - super().__init__(cloud_build_config=BuildConfig(image="my-custom-image")) # no cloud compute, for example default work diff --git a/docs/source-app/glossary/command_lines/command_lines.rst b/docs/source-app/glossary/command_lines/command_lines.rst deleted file mode 100644 index 15d77272c1692..0000000000000 --- a/docs/source-app/glossary/command_lines/command_lines.rst +++ /dev/null @@ -1,69 +0,0 @@ -:orphan: - -############################ -Command-line Interface (CLI) -############################ - -**Audience:** Users looking to create a command line interface (CLI) for their application. - ----- - -************** -What is a CLI? -************** - -A Command-line Interface (CLI) is an user interface (UI) in a terminal to interact with a specific program. - -.. note:: - - The Lightning guideline to build CLI is `lightning_app ...` or ` ...`. - -As an example, Lightning provides a CLI to interact with your Lightning Apps and the `lightning.ai `_ platform as follows: - -.. code-block:: bash - - main - ├── fork - Forks an App. - ├── init - Initializes a Lightning App and/or Component. - │ ├── app - │ ├── component - │ ├── pl-app - Creates an App from your PyTorch Lightning source files. - │ └── react-ui - Creates a React UI to give a Lightning Component a React.js web UI - ├── install - Installs a Lightning App and/or Component. - │ ├── app - │ └── component - ├── list - Lists Lightning AI self-managed resources (apps) - │ └── apps - Lists your Lightning AI Apps. - ├── login - Logs in to your lightning.ai account. - ├── logout - Logs out of your lightning.ai account. - ├── run - Runs a Lightning App locally or on the cloud. - │ └── app - Runs an App from a file. - ├── show - Shows given resource. - │ └── logs - Shows cloud application logs. By default prints logs for all currently available Components. - ├── stop - Stops your App. - └── tree - Shows the command tree of your CLI. - -Learn more about `Command-line interfaces here `_. - ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a Command Line Interface - :description: Learn how to develop a CLI for your App. - :col_css: col-md-6 - :button_link: ../../workflows/build_command_line_interface/index_content.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/dag.rst b/docs/source-app/glossary/dag.rst deleted file mode 100644 index 9578b1e5260a8..0000000000000 --- a/docs/source-app/glossary/dag.rst +++ /dev/null @@ -1,46 +0,0 @@ -###################### -Directed Acyclic Graph -###################### -**Audience:** Users coming from MLOps to Lightning Apps, looking for more flexibility. - ----- - -***************************** -Is Lightning a DAG framework? -***************************** -No. - -A Lightning App enables developers to express complex, interactive applications that are impossible to create with DAGs. - ----- - -********************************* -Can I Build a DAG with Lightning? -********************************* -Yes! - -DAGs are one of the easiest Lightning Apps to build. For example, here's a :doc:`full app that defines a DAG <../examples/dag/dag>`. - ----- - -******** -Examples -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a DAG - :description: Learn how to create a DAG with Lightning - :col_css: col-md-4 - :button_link: ../examples/dag/dag.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/debug_app.rst b/docs/source-app/glossary/debug_app.rst deleted file mode 100644 index 2d5c0d19903b0..0000000000000 --- a/docs/source-app/glossary/debug_app.rst +++ /dev/null @@ -1,3 +0,0 @@ -:orphan: - -.. include:: ../workflows/debug_locally.rst diff --git a/docs/source-app/glossary/distributed_fe.rst b/docs/source-app/glossary/distributed_fe.rst deleted file mode 100644 index 36d64b01436b6..0000000000000 --- a/docs/source-app/glossary/distributed_fe.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -##################### -Distributed Front-End -##################### diff --git a/docs/source-app/glossary/distributed_hardware.rst b/docs/source-app/glossary/distributed_hardware.rst deleted file mode 100644 index 0a64f5f5c0720..0000000000000 --- a/docs/source-app/glossary/distributed_hardware.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -#################### -Distributed Hardware -#################### diff --git a/docs/source-app/glossary/environment_variables.rst b/docs/source-app/glossary/environment_variables.rst deleted file mode 100644 index dcbcd8c6f4ab7..0000000000000 --- a/docs/source-app/glossary/environment_variables.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _environment_variables: - -********************* -Environment Variables -********************* - -If your App is using configuration values you don't want to commit with your App source code, you can use environment variables. - -Lightning allows you to set environment variables when running the App from the CLI with the `lightning_app run app` command. You can use environment variables to pass any values to the App, and avoiding sticking those values in the source code. - -Set one or multiple variables using the **--env** option: - -.. code:: bash - - lightning_app run app app.py --cloud --env FOO=BAR --env BAZ=FAZ - -Environment variables are available in all Flows and Works, and can be accessed as follows: - -.. code:: python - - import os - - print(os.environ["FOO"]) # BAR - print(os.environ["BAZ"]) # FAZ - -.. note:: - Environment variables are not encrypted. For sensitive values, we recommend using :ref:`Encrypted Secrets `. diff --git a/docs/source-app/glossary/event_loop.rst b/docs/source-app/glossary/event_loop.rst deleted file mode 100644 index 30d1bd3b3acfd..0000000000000 --- a/docs/source-app/glossary/event_loop.rst +++ /dev/null @@ -1,11 +0,0 @@ -########## -Event loop -########## - -Drawing inspiration from modern web frameworks like `React.js `_, the Lightning App runs all flows in an **event loop** (forever), which is triggered several times a second after collecting any works' state change. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_loop.gif - -When running a Lightning App in the cloud, the ``LightningWork`` run on different machines. LightningWork communicates any state changes to the **event loop** which re-executes the flow with the newly-collected works' state. - -.. _app_event_loop: diff --git a/docs/source-app/glossary/fault_tolerance.rst b/docs/source-app/glossary/fault_tolerance.rst deleted file mode 100644 index b0ee6dfd21102..0000000000000 --- a/docs/source-app/glossary/fault_tolerance.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -############### -Fault tolerance -############### - -.. note:: documentation under construction diff --git a/docs/source-app/glossary/index.rst b/docs/source-app/glossary/index.rst deleted file mode 100644 index 50e4c1be435d8..0000000000000 --- a/docs/source-app/glossary/index.rst +++ /dev/null @@ -1,155 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :hidden: - - ios_and_android - app_tree - build_config/build_config - command_lines/command_lines - dag - event_loop - environment_variables - secrets - front ends <../workflows/add_web_ui/glossary_front_end> - Lightning app <../core_api/lightning_app/index> - sharing_components - scheduling - storage/storage - restful_api/restful_api - add web ui <../workflows/add_web_ui/glossary_ui> - use_local_lightning - -######## -Glossary -######## - -.. raw:: html - -
-
- -.. displayitem:: - :header: Android Lightning App - :description: Use Lightning with android apps. - :col_css: col-md-12 - :button_link: ios_and_android.html - :height: 100 - -.. displayitem:: - :header: App Components Tree - :description: Learn how components can be nested to form component trees where the LightningFlows are its branches and LightningWorks are its leaves. - :col_css: col-md-12 - :button_link: app_tree.html - :height: 100 - -.. displayitem:: - :header: Build Configuration - :description: Prepare your requirements, add custom build commands or use docker image - :col_css: col-md-12 - :button_link: build_config/build_config.html - :height: 100 - -.. displayitem:: - :header: Command Line Interface (CLI) - :description: Learn about the Lightning CLI - :col_css: col-md-12 - :button_link: command_lines/command_lines.html - :height: 100 - -.. displayitem:: - :header: DAG - :description: Learn about directed acyclic graph, their properties and usage - :col_css: col-md-12 - :button_link: dag.html - :height: 100 - -.. displayitem:: - :header: Event Loop - :description: Learn how the Infinite Event Loop enables high distributed reactivity by triggering after collecting state changes. - :col_css: col-md-12 - :button_link: event_loop.html - :height: 100 - -.. displayitem:: - :header: Environment Variables - :description: Add secrets such as API keys or access tokens - :col_css: col-md-12 - :button_link: environment_variables.html - :height: 100 - -.. displayitem:: - :header: Encrypted Secrets - :description: Learn how to add passwords to your Lightning apps - :col_css: col-md-12 - :button_link: secrets.html - :height: 100 - -.. displayitem:: - :header: Frontend - :description: Customize your App View with any framework you want - :col_css: col-md-12 - :button_link: ../workflows/add_web_ui/glossary_front_end.html - :height: 100 - -.. displayitem:: - :header: iOS Lightning App - :description: Use Lightning with iOS apps. - :col_css: col-md-12 - :button_link: ios_and_android.html - :height: 100 - -.. displayitem:: - :header: Lightning App - :description: A Lightning app is a collection of connected components that form a workflow - :col_css: col-md-12 - :button_link: ../core_api/lightning_app/index.html - :height: 100 - -.. displayitem:: - :header: Mounts - :description: Mount Cloud Data - :col_css: col-md-12 - :button_link: mount.html - :height: 100 - -.. displayitem:: - :header: Sharing Components - :description: Let's create an ecosystem altogether - :col_css: col-md-12 - :button_link: sharing_components.html - :height: 100 - -.. displayitem:: - :header: Scheduling - :description: Orchestrate execution at specific times - :col_css: col-md-12 - :button_link: scheduling.html - :height: 100 - -.. displayitem:: - :header: Storage - :description: Easily share files even across multiple machines - :col_css: col-md-12 - :button_link: storage/storage.html - :height: 100 - -.. displayitem:: - :header: REST API - :description: Learn how to set up a RESTful API endpoint - :col_css: col-md-12 - :button_link: restful_api/restful_api.html - :height: 100 - -.. displayitem:: - :header: UI - :description: Combine multiple frameworks to create your own UI - :col_css: col-md-12 - :button_link: ../workflows/add_web_ui/glossary_ui.html - :height: 100 - -.. displayitem:: - :header: Using a development branch of Lightning on the Cloud - :description: Learn how to contribute to the Lightning framework in the cloud - :col_css: col-md-12 - :button_link: use_local_lightning.html - :height: 100 diff --git a/docs/source-app/glossary/ios_and_android.rst b/docs/source-app/glossary/ios_and_android.rst deleted file mode 100644 index 90aeecbc0b141..0000000000000 --- a/docs/source-app/glossary/ios_and_android.rst +++ /dev/null @@ -1,26 +0,0 @@ - -############################################### -Apple and Android mobile devices with Lightning -############################################### - -Audience: Users who want to develop Lightning Apps for Apple or Android mobile devices. - ----- - -*********************************************************** -Develop a Lightning App for Apple or Android mobile devices -*********************************************************** - -There are a couple of ways you can go about building Lightning Apps that work on Apple or Android mobile devices. - -Option 1 -^^^^^^^^ - -You can develop a Lightning App that interacts with an iOS or Android app. -The ML and backend services live on the Lightning App, but the iOS or Android code (obj-c/swift or android) lives on the mobile devices. - -Option 2 -^^^^^^^^ - -You can build a mobile-first React Lightning App that works on both Apple and Android mobile devices. -The `InVideo app `_ is a good example of a Lightning App that does just that. diff --git a/docs/source-app/glossary/lightning_app_overview/index.rst b/docs/source-app/glossary/lightning_app_overview/index.rst deleted file mode 100644 index 09de273affc50..0000000000000 --- a/docs/source-app/glossary/lightning_app_overview/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -########################### -Lightning Apps Key concepts -########################### - -**Audience:** Users who want to know how the 🤯 magic works under the hood. - ----- - -.. note:: This page is under construction diff --git a/docs/source-app/glossary/mount.rst b/docs/source-app/glossary/mount.rst deleted file mode 100644 index a62d72b5b798d..0000000000000 --- a/docs/source-app/glossary/mount.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../workflows/mount_cloud_object_store.rst diff --git a/docs/source-app/glossary/restful_api/restful_api.rst b/docs/source-app/glossary/restful_api/restful_api.rst deleted file mode 100644 index 6e04f60c75f1c..0000000000000 --- a/docs/source-app/glossary/restful_api/restful_api.rst +++ /dev/null @@ -1,53 +0,0 @@ -:orphan: - -########### -RESTful API -########### - -**Audience:** Users looking to create an API in their App to allow users to activate functionalities from external sources. - ----- - -********************** -What is a RESTful API? -********************** - -A RESTful API is a set of external URL routes exposed by a server that enables clients to trigger some functionalities, such as getting or putting some data, uploading files, etc.. - -This provides great flexibility for users as they can easily discover functionalities made available by the App Builders. - -The Lightning App framework supports the four primary HTTP methods: `GET`, `POST`, `PUT`, `DELETE`. - -These methods are guidelines to organize your RESTful Services and help users understand your functionalities. - -* **`GET`:** Reads data from the server. -* **`POST`:** Creates new resources. -* **`PUT`:** Updates/replaces existing resources. -* **`DELETE`:** Deletes resources. - -Learn more about `HTTP Methods for RESTful Services here `_. - -The Lightning App framework uses the popular `FastAPI `_ and `Pydantic `_ frameworks under the hood. This means you can use all their features while building your App. - ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a RESTful API - :description: Learn how to develop an API for your App. - :col_css: col-md-6 - :button_link: ../../workflows/build_rest_api/index_content.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/scheduling.rst b/docs/source-app/glossary/scheduling.rst deleted file mode 100644 index 0e04dc35c897d..0000000000000 --- a/docs/source-app/glossary/scheduling.rst +++ /dev/null @@ -1,185 +0,0 @@ -:orphan: - -########## -Scheduling -########## - -The Lightning Scheduling system makes it easy to schedule your components execution with any arbitrary conditions. - - ----- - -************************ -Schedule your components -************************ - -The LightningFlow has a ``schedule`` method which can be used to schedule your components. - -.. code-block:: python - - from lightning.app import LightningWork, LightningFlow - from lightning.app.storage import Path - - - class MyFlow(LightningFlow): - - def run(self): - if self.schedule("hourly"): - # run some code once every hour. - - if self.schedule("daily"): - # run some code once day. - - if self.schedule("daily") and anything_else: - # run some code once day if the anything else is also True. - - if self.schedule("2 4 * * mon,fri"): - # defined with cron syntax, run some code at 04:02 on every Monday and Friday. - -Learn more about the cron syntax `here `_ - ----- - -************** -Best Practices -************** - -In the example above, the line ``self.schedule("hourly")`` will return ``True`` for a **single** flow execution every hour. Mathematically, this is known as a dirac. - -1. Instantiate your component under the schedule method and run outside as follows: - -.. code-block:: python - - from lightning.app import LightningFlow - from lightning.app.structures import List - - class ScheduledDAG(LightningFlow): - def __init__(self): - super().__init__() - self.list = List() - - def run(self): - if self.schedule("hourly"): - # dynamically instantiate - # don't forget to always attach - # your components to the flow !!! - self.list.append(MyDAGFlow(...)) - - # run all dags, but the completed ones - # are cached and don't re-execute. - for dag in self.list: - dag.run() - - -2. Run a single work under the schedule with different arguments to have it re-run. - -.. code-block:: python - - from lightning.app import LightningFlow - from time import time - - class ScheduledDAG(LightningFlow): - def __init__(self): - super().__init__() - self.data_processor = DataProcessorWork(...) - - def run(self): - ... - if self.schedule("hourly"): - self.data_processor.run(trigger_time=time()) - - -3. Capture the event in the state and execute your sequential works outside. - -.. code-block:: python - - from lightning.app import LightningFlow - from time import time - - class ScheduledDAG(LightningFlow): - def __init__(self): - super().__init__() - self.should_execute = False - self.data_processor = DataProcessorWork(...) - self.training_work = KerasTrainingWork(...) - - def run(self): - ... - if self.schedule("hourly"): - self.should_execute = True - - # Runs in 10 min - if self.should_execute: - # Runs in 5 min - self.data_processor.run(trigger_time=time()) - if self.data_processor.has_succeeded: - # Runs in 5 min - self.training_work.run(self.data_processor.data) - if self.training_work.has_succeeded: - self.should_execute = False - ----- - -*********** -Limitations -*********** - -As stated above, the schedule acts as a dirac and is **True** for a single flow execution. -Therefore, sequential works execution under the schedule won't work as they don't complete within a single flow execution. - -Here is an example of something which **WON'T** work: - -.. code-block:: python - - from lightning.app import LightningFlow - from time import time - - class ScheduledDAG(LightningFlow): - def __init__(self): - super().__init__() - self.data_processor = DataProcessorWork(...) - self.training_work = KerasTrainingWork(...) - - def run(self): - ... - if self.schedule("hourly"): - # This finishes 5 min later - self.data_processor.run(trigger_time=time()) - if self.data_processor.has_succeeded: - # This will never be reached as the - # data processor will keep processing forever... - self.training_work.run(self.data_processor.data) - ----- - -************************** -Frequently Asked Questions -************************** - -- **Q: Can I use multiple nested scheduler?** No, as they might cancel themselves out, but you can capture the event of one to trigger the next one. - -- **Q: Can I use any arbitrary logic to schedule?** Yes, this design enables absolute flexibility, but you need to be careful to avoid bad practices. - ----- - -******** -Examples -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a DAG - :description: Learn how to schedule a DAG execution - :col_css: col-md-4 - :button_link: ../examples/dag/dag.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/secrets.rst b/docs/source-app/glossary/secrets.rst deleted file mode 100644 index 95a0d564c648d..0000000000000 --- a/docs/source-app/glossary/secrets.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. _secrets: - -################# -Encrypted Secrets -################# - -Encrypted Secrets allow you to pass private data to your apps, like API keys, access tokens, database passwords, or other credentials, in a secure way without exposing them in your code. -Secrets provide you with a secure way to store this data in a way that is accessible to Apps so that they can authenticate third-party services/solutions. - -.. tip:: - For non-sensitive configuration values, we recommend using :ref:`plain-text Environment Variables `. - -************ -Add a secret -************ - -Add the secret to your profile on lightning.ai. -Log in to your lightning.ai account > **Profile** > **Secrets** tab > click the **+New** button. -Provide a name and value to your secret, for example, name could be "github_api_token". - -.. note:: - Secret names must start with a letter and can only contain letters, numbers, dashes, and periods. The Secret names must comply with `RFC1123 naming conventions `_. The Secret value has no restrictions. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning//encrypted_secrets_login.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning//encrypted_secrets_login.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -************ -Use a secret -************ - -1. Add an environment variable to your app to read the secret. For example, add an "api_token" environment variable: - -.. code:: python - - import os - - component.connect(api_token=os.environ["api_token"]) - -2. Pass the secret to your app run with the following command: - -.. code:: bash - - lightning_app run app app.py --cloud --secret = - -In this example, the command would be: - -.. code:: bash - - lightning_app run app app.py --cloud --secret api_token=github_api_token - - -The ``--secret`` option can be used for multiple Secrets, and alongside the ``--env`` option. - -Here's an example: - -.. code:: bash - - lightning_app run app app.py --cloud --env FOO=bar --secret MY_APP_SECRET=my-secret --secret ANOTHER_SECRET=another-secret - - ----- - -****************** -How does this work -****************** - -When a Lightning App (App) **runs in the cloud**, a Secret can be exposed to the App using environment variables. -The value of the Secret is encrypted in the Lightning.ai database, and is only decrypted and accessible to -LightningFlow (Flow) or LightningWork (Work) processes in the cloud (when you use the ``--cloud`` option running your App). diff --git a/docs/source-app/glossary/sharing_components.rst b/docs/source-app/glossary/sharing_components.rst deleted file mode 100644 index 2426bb43d4469..0000000000000 --- a/docs/source-app/glossary/sharing_components.rst +++ /dev/null @@ -1,50 +0,0 @@ -##################### -Sharing my components -##################### - -**Audience:** Users who want to know how to share component. - -**Level:** Basic - ----- - -******************************************** -Why should I consider sharing my components? -******************************************** - -Lightning is community driven and its core objective is to make AI accessible to everyone. - -By creating components and sharing them with everyone else, the barrier to entry will go down. - ----- - -************************************ -How should I organize my components? -************************************ - -By design, Lightning components are nested to form component trees where the ``LightningFlows`` are its branches and ``LightningWorks`` are its leaves. - -This design has two primary advantages: - -* This helps users organize and maintain their code with more ease. -* This also helps create an ecosystem with **reusable** components. - - -Now, imagine you have implemented a **KerasScriptRunner** component for training any `Keras `_ model with `Tensorboard UI `_ integrated. - -Here are the best practices steps before sharing the component: - -* **Testing**: Ensure your component is well tested by following the :doc:`../testing` guide. -* **Documented**: Ensure your component has a docstring and comes with some usage explications. - -.. Note:: As a Lightning user, it helps to implement your components thinking someone else is going to use them. - ----- - -***************************************** -How should I proceed to share components? -***************************************** - -Once your component is ready, create a *PiPy* package with your own library and then it can be reused by anyone else. - -Here is a `Component Template `_ from `William Falcon `_ to guide your component. diff --git a/docs/source-app/glossary/storage/differences.rst b/docs/source-app/glossary/storage/differences.rst deleted file mode 100644 index ed45edd069632..0000000000000 --- a/docs/source-app/glossary/storage/differences.rst +++ /dev/null @@ -1,78 +0,0 @@ -:orphan: - -################################## -Differences between Drive and Path -################################## - -**Audience:** Users who want to share files between components. - - -The Lightning Storage system makes it easy to share files between LightningWork so you can run your app both locally and in the cloud without changing the code. - - -Lightning storage provides two solutions :class:`~lightning.app.storage.drive.Drive` and :class:`~lightning.app.storage.path.Path` to deal with files locally and in the cloud likewise. - - ----- - -***************** -What is a Drive ? -***************** - -The Drive object provides a central place for your components to share data. - -The drive acts as an isolate folder and any component can access it by knowing its name. - -Your components can put, list, get, delete files from and to the Drive (except LightningFlow's). - ----- - -**************** -What is a Path ? -**************** - -The Path object is a reference to a specific file or directory from a LightningWork and can be used to transfer those files to another LightningWork (one way, from source to destination). - -A good mental representation of the Path Object usage is the `relay race `_. -To make a transfer, the LightningWork Receiver asks (e.g when the path object is passed by the flow to the Receiver) -for a copy of the files (baton) owned by their LightningWork Producer (e.g the work that created the files). - ----- - -********************************* -When should I use Drive vs Path ? -********************************* - -The Drive should be used when you want to easily share data between components but the Path enables to create cleaner shareable -component where you want to exposes some files to be transferred (like an HPO component sharing the best model weights) for anyone else to use. - -The Drive is more intuitive and easier to get on-boarded with, but in more advanced use cases, you might appreciate the Path Object -which makes uni-directional files transfer simpler. - ----- - -.. raw:: html - -
-
- -.. displayitem:: - :header: The Drive Object. - :description: Put, List and Get Files From a Shared Drive Disk. - :col_css: col-md-4 - :button_link: drive.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: The Path Object. - :description: Transfer Files From One Component to Another by Reference. - :col_css: col-md-4 - :button_link: path.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/storage/drive.rst b/docs/source-app/glossary/storage/drive.rst deleted file mode 100644 index dffdb979f305a..0000000000000 --- a/docs/source-app/glossary/storage/drive.rst +++ /dev/null @@ -1,13 +0,0 @@ -:orphan: - -.. _drive_storage: - -############# -Drive Storage -############# - -**Audience:** Users who want to put, list, and get files from a shared disk space. - ----- - -.. include:: ../../glossary/storage/drive_content_old.rst diff --git a/docs/source-app/glossary/storage/drive_content.rst b/docs/source-app/glossary/storage/drive_content.rst deleted file mode 100644 index ff1540e8ce6f6..0000000000000 --- a/docs/source-app/glossary/storage/drive_content.rst +++ /dev/null @@ -1,223 +0,0 @@ -:orphan: - -************************** -What are Lightning Drives? -************************** - -Lightning Drives are shared app storage that allow you to share files between :doc:`LightningWork (Work) <../../core_api/lightning_work/index>` components, so that you distributed components can share files when running on the cloud. Using drives, you can run your Lightning App both locally and in the cloud without changing the code. - -The Drive object provides a central place for your components to share data. - -The Drive acts as an isolated folder and any component can access it by knowing its name. - -We currently support two types of Drives: Lightning-managed (``lit://``) and S3 (``s3://``). - -+-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------+ -| Lightning-managed (``lit://``) | Allows read-write operations and are accessible through the Drive API from a Work. | -| | | -| | They allow your components to put, list, get, and delete files from and to the Drive (except LightningFlows). | -+-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------+ -| S3 (``s3://``) | S3 is AWS S3 storage mounted at a filesystem mount point. S3 is read-only (for now) and its primary purpose is | -| | to give you a permanent location to access your training data. | -| | | -| | They allow your components to list and get files located on the Drive. | -+-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------+ - ----- - -********************** -What Drives do for you -********************** - -Think of every instance of the Drive object acting like a Google Drive or like Dropbox. - -By sharing the Drive between components through the LightningFlow, -several components can have a shared place to read (S3 Drives) or read and write (Lightning-managed Drives) files from. - -S3 Drive Limitations -^^^^^^^^^^^^^^^^^^^^ - -These limitations only apply to S3 Drives: - -* There is no top level “shareable” S3 drive object. Each S3 Drive is owned by a particular Work. However, it’s possible to create a Drive with the same location across multiple Works. - -* S3 buckets cannot be mounted as Drives once a Work has been instantiated. The `Drive` object must be initialized passed to a Work at creation time. - -* Whenever a Drive is mounted to a Work, an indexing process will be done again for the provided S3 bucket. This may lead to performance issues with particularly large S3 buckets. For context, 1M files with 2-3 levels of nesting takes less than 1 second to index. - ----- - -************** -Create a Drive -************** - -In order to create a Drive, you simply need to pass its name with the prefix ``lit://`` or ``s3://``. - -.. note:: We do not support mounting single objects for S3 buckets, so there must be a trailing `/` in the s3:// URL. For example: ``s3://foo/bar/``. - -.. code-block:: python - - from lightning.app.storage import Drive - - # The identifier of this Drive is ``drive_1`` - # Note: You need to add Lightning protocol ``lit://`` as a prefix. - - drive_1 = Drive("lit://drive_1") - - # The identifier of this Drive is ``drive_2`` - drive_2 = Drive("s3://drive_2/") - -Any component can create a drive object for ``lit://`` Drives. - -.. code-block:: python - - from lightning.app import LightningFlow, LightningWork - from lightning.app.storage import Drive - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - - def run(self): - ... - - - class Work(LightningWork): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - - def run(self): - ... - ----- - -***************************** -Supported actions with Drives -***************************** - -A Lightning-managed Drive supports put, list, get, and delete actions. - -An S3 Drive supports list and get actions (for now). - -.. code-block:: python - - from lightning.app.storage import Drive - - drive = Drive("lit://drive") - - drive.list(".") # Returns [] as empty - - # Created file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - drive.put("a.txt") - - drive.list(".") # Returns ["a.txt"] as the file copied in the Drive during the put action. - - drive.get("a.txt") # Get the file into the current worker - - drive.delete("a.txt") - - drive.list(".") # Returns [] as empty - ----- - -********************************** -Component interactions with Drives -********************************** - -Here is an illustrated code example on how to create drives within Works. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/drive_2.png - -.. code-block:: python - - from lightning.app import LightningFlow, LightningWork, LightningApp - from lightning.app.storage import Drive - - - class Work_A(LightningWork): - def __init__(self): - super().__init__() - # The identifier of the Drive is ``drive_1`` - # Note: You need to add Lightning protocol ``lit://`` as a prefix. - self.drive_1 = Drive("lit://drive_1") - - def run(self): - # 1. Create a file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - self.drive_1.put("a.txt") - - - class Work_B(LightningWork): - def __init__(self): - super().__init__() - - # Note: Work B has access 2 drives. - - # The identifier of this Drive is ``drive_1`` - self.drive_1 = Drive("lit://drive_1") - # The identifier of this Drive is ``drive_2`` - self.drive_2 = Drive("lit://drive_2") - - def run(self): - # 1. Create a file. - with open("b.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into both drives. - self.drive_1.put("b.txt") - self.drive_2.put("b.txt") - - - class Work_C(LightningWork): - def __init__(self): - super().__init__() - self.drive_2 = Drive("lit://drive_2") - - def run(self): - # 1. Create a file. - with open("c.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - self.drive_2.put("c.txt") - ----- - -************************* -Transfer files with Drive -************************* - -In the example below, the Drive is created by the Flow and passed to its Works. - -The ``Work_1`` put a file **a.txt** in the **Drive("lit://this_drive_id")** and the ``Work_2`` can list and get the **a.txt** file from it. - -.. literalinclude:: ../../../../examples/app/drive/app.py - ----- - -.. raw:: html - -
-
- -.. displayitem:: - :header: Learn about the Path Object. - :description: Transfer Files From One Component to Another by Reference. - :col_css: col-md-4 - :button_link: path.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/storage/drive_content_old.rst b/docs/source-app/glossary/storage/drive_content_old.rst deleted file mode 100644 index 3c37f883fd31d..0000000000000 --- a/docs/source-app/glossary/storage/drive_content_old.rst +++ /dev/null @@ -1,199 +0,0 @@ -:orphan: - - -************ -About Drives -************ - -Lightning Drive storage makes it easy to share files between LightningWorks so you can run your Lightning App both locally and in the cloud without changing the code. - -The Drive object provides a central place for your components to share data. - -The Drive acts as an isolate folder and any component can access it by knowing its name. - -Your components can put, list, get, and delete files from and to the Drive (except LightningFlows). - ----- - -*********************** -What Drive does for you -*********************** - -Think of every instance of the Drive object acting like a Google Drive or like Dropbox. - -By sharing the Drive between components through the LightningFlow, -several components can have a shared place to read and write files from. - ----- - -************** -Create a Drive -************** - -In order to create a Drive, you simply need to pass its name with the prefix ``lit://`` as follows: - -.. code-block:: python - - from lightning.app.storage import Drive - - # The identifier of this Drive is ``drive_1`` - # Note: You need to add Lightning protocol ``lit://`` as a prefix. - - drive_1 = Drive("lit://drive_1") - - # The identifier of this Drive is ``drive_2`` - drive_2 = Drive("lit://drive_2") - -Any components can create a drive object. - -.. code-block:: python - - from lightning.app import LightningFlow, LightningWork - from lightning.app.storage import Drive - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - - def run(self): - ... - - - class Work(LightningWork): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - - def run(self): - ... - ----- - -***************************** -Supported actions with Drives -***************************** - -A Drive supports put, list, get, and delete actions. - -.. code-block:: python - - from lightning.app.storage import Drive - - drive = Drive("lit://drive") - - drive.list(".") # Returns [] as empty - - # Created file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - drive.put("a.txt") - - drive.list(".") # Returns ["a.txt"] as the file copied in the Drive during the put action. - - drive.get("a.txt") # Get the file into the current worker - - drive.delete("a.txt") - - drive.list(".") # Returns [] as empty - ----- - -********************************** -Component interactions with Drives -********************************** - -Here is an illustrated code example on how to create drives within works. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/drive_2.png - -.. code-block:: python - - from lightning.app import LightningFlow, LightningWork, LightningApp - from lightning.app.storage import Drive - - - class Work_A(LightningWork): - def __init__(self): - super().__init__() - # The identifier of the Drive is ``drive_1`` - # Note: You need to add Lightning protocol ``lit://`` as a prefix. - self.drive_1 = Drive("lit://drive_1") - - def run(self): - # 1. Create a file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - self.drive_1.put("a.txt") - - - class Work_B(LightningWork): - def __init__(self): - super().__init__() - - # Note: Work B has access 2 drives. - - # The identifier of this Drive is ``drive_1`` - self.drive_1 = Drive("lit://drive_1") - # The identifier of this Drive is ``drive_2`` - self.drive_2 = Drive("lit://drive_2") - - def run(self): - # 1. Create a file. - with open("b.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into both drives. - self.drive_1.put("b.txt") - self.drive_2.put("b.txt") - - - class Work_C(LightningWork): - def __init__(self): - super().__init__() - self.drive_2 = Drive("lit://drive_2") - - def run(self): - # 1. Create a file. - with open("c.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - self.drive_2.put("c.txt") - ----- - -***************************** -Transfer files with Drive -***************************** - -In the example below, the Drive is created by the flow and passed to its LightningWork's. - -The ``Work_1`` put a file **a.txt** in the **Drive("lit://this_drive_id")** and the ``Work_2`` can list and get the **a.txt** file from it. - -.. literalinclude:: ../../../../examples/app/drive/app.py - - ----- - -.. raw:: html - -
-
- -.. displayitem:: - :header: Learn about the Path Object. - :description: Transfer Files From One Component to Another by Reference. - :col_css: col-md-4 - :button_link: path.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/storage/path.rst b/docs/source-app/glossary/storage/path.rst deleted file mode 100644 index 4cf41c1453aa4..0000000000000 --- a/docs/source-app/glossary/storage/path.rst +++ /dev/null @@ -1,326 +0,0 @@ -:orphan: - -############ -Path Storage -############ - -**Audience:** Users who want to share files between components. - - -The Lightning Storage system makes it easy to share files between LightningWork so you can run your app both locally and in the cloud without changing the code. - ----- - -*********************** -What is a Path Object ? -*********************** - -The Path object is a reference to a specific file or directory from a LightningWork and can be used to transfer those files to another LightningWork (one way, from source to destination). - -A good mental representation of the Path Object usage is the `relay race `_. -To make a transfer, the receiver asks (e.g when the path object is passed by the flow to the receiver) -for a copy of the files (baton) owned by their producer (e.g the LightningWork which created the files). - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/path2.png - -******************************************* -How does the Path Object works internally ? -******************************************* - -To understand the Path Object internal, let's first answer this question: How do you locate a specific file or folder within a distributed system made of multiple machines ? - -You need to know on which machine the file or folder is located (e.g the LightningWork name uniquely identify its own machine in the cloud) and -then you need the local path of the file or folder on that machine. - -In simple words, the Lightning Path augments :class:`pathlib.Path` object by tracking on which machine the file or folder is located. - ----- - -************************** -When to use Path storage ? -************************** - -In the cloud, every :class:`~lightning.app.core.work.LightningWork` runs in a separate machine with its own filesystem. -This means files in one Work cannot be directly accessed in another like you would be able to when running the app locally. -But with Lightning Storage, this is easy: Simply declare which files need to be shared and Lightning will take care of the rest. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/path.mp4 - :width: 600 - :autoplay: - :loop: - :muted: - - ----- - - -*********************************** -Tell Lightning where your files are -*********************************** - -Convert every filesystem path you want to share with other LightningWorks to by adding ``lit://`` in front of it. - -.. code-block:: python - - from lightning.app import LightningWork - from lightning.app.storage import Path - - - class SourceWork(LightningWork): - def __init__(self): - super().__init__() - self.checkpoint_dir = None - - def run(self): - # Normally you would do: - # self.checkpoint_dir = "outputs/checkpoints" - # os.makedirs("outputs/checkpoints") - # ... - - # In Lightning, do: - self.checkpoint_dir = "lit://outputs/checkpoints" - os.makedirs(self.checkpoint_dir) - ... - - -Under the hood, we convert this string to a :class:`~lightning.app.storage.path.Path` object, which is a drop-in replacement for :class:`pathlib.Path` meaning it will work with :mod:`os`, :mod:`os.path` and :mod:`pathlib` filesystem operations out of the box! - - ----- - - -**************************** -Access files in another Work -**************************** - -Accessing files from another LightningWork is as easy as handing the path over by reference. -For example, share a directory by passing it as an input to the run method of the destination work: - -.. code-block:: python - :emphasize-lines: 12 - - from lightning.app import LightningFlow - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.source = SourceWork() - self.destination = DestinationWork() - - def run(self): - self.source.run() - # Pass the Path reference from one work to another - self.destination.run(self.source.checkpoint_dir) - - -When the destination Work starts, Lightning will automatically transfer the files to its filesystem (if they exist on the other end): - -.. code-block:: python - - class DestinationWork(LightningWork): - def run(self, checkpoint_dir): - # The directory is now accessible inside this Work - files = os.listdir(checkpoint_dir) - ... - - -The automatic transfer only happens if the referenced files already exist in the originating LightningWork and it will overwrite any files that already exist locally. -In all other cases, you can trigger the transfer manually. - - ----- - - -****************** -Get files manually -****************** - -If you need to access files at a specific time or transfer them multiple times, use ``.get()`` method: - -.. code-block:: python - - def run(self, checkpoint_dir): - ... - # Make the directory available - checkpoint_dir.get() - - # If the path already exists locally, you can force overwriting it - checkpoint_dir.get(overwrite=True) - - files = os.listdir(checkpoint_dir) - ... - - -Multiple calls to the ``.get()`` method will always result in file transfers, regardless of whether the files have changed or not. -If the path does not exist remotely, it will raise a ``FileNotFoundError``. -If you need to handle this case, the Path also offers a method to check if files exist remotely. - ----- - - -******************************** -Check if a file or folder exists -******************************** - -You can check if a path exists locally or remotely in the source Work using the ``.exists_local()`` and ``.exists_remote()`` methods: - -.. code-block:: python - - def run(self, checkpoint_dir): - if checkpoint_dir.exists_remote(): - # Get the file only if it exists in the source Work - checkpoint_dir.get() - - # OR - - if checkpoint_dir.exists_local(): - # Do something with the file if it exists locally - files = os.listdir(checkpoint_dir) - - ----- - - -************* -Persist files -************* - -If a LightningWork finishes or stops due to an interruption (e.g., due to insufficient credits), the filesystem and all files in it get deleted (unless running locally). -Lightning makes sure all Paths that are part of the state get stored and made accessible to the other Works that still need these files. - -.. code-block:: python - - from lightning.app.storage import Path - - - class Work(LightningWork): - def __init__(self): - super().__init__() - # The files in this path will be saved as an artifact when the Work finishes - self.checkpoint_dir = "lit://outputs/checkpoints" - - # The files in this path WON'T be saved because it is not declared as a Lightning Path - self.log_dir = "outputs/logs" - - ----- - - -********************************* -Example: Share a model checkpoint -********************************* - -A common workflow in ML is to use a checkpoint created by another component. -First, define a component that saves a checkpoint: - -.. code:: python - :emphasize-lines: 14-18 - - from lightning.app import LightningFlow, LightningWork - from lightning.app.storage import Path - import torch - import os - - - class ModelTraining(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.checkpoint_dir = "lit://outputs/checkpoints" - - def run(self): - # create a directory - os.makedirs(self.model_checkpoints_path, exist_ok=True) - # make fake checkpoints - checkpoint_1 = torch.tensor([0, 1, 2, 3, 4]) - checkpoint_2 = torch.tensor([0, 1, 2, 3, 4]) - torch.save(checkpoint_1, os.path.join(self.checkpoint_dir, "checkpoint_1.ckpt")) - torch.save(checkpoint_2, os.path.join(self.checkpoint_dir, "checkpoint_2.ckpt")) - - -Next, define a component that needs the checkpoints: - -.. code:: python - :emphasize-lines: 4, 7 - - class ModelDeploy(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__() - - def run(self, checkpoint_dir): - ckpts = os.listdir(checkpoint_dir) - checkpoint_1 = torch.load(ckpts[0]) - checkpoint_2 = torch.load(ckpts[1]) - -Link both components via a parent component: - -.. code:: python - :emphasize-lines: 7 - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.train = ModelTraining() - - # pass the checkpoint path - self.deploy = ModelDeploy() - - def run(self): - self.train.run() - self.deploy.run(checkpoint_dir=self.train.checkpoint_dir) - - - app = L.LightningApp(Flow()) - - ----- - -************************** -Frequently Asked Questions -************************** - -- **Q: Can files in a LightningWork be accessed inside the LightningFlow too?** - - No, LightningFlow is intentionally designed not to perform filesystem operations and computations and is intended to exclusively orchestrate Flow and Work. - -- **Q: Is it possible to reference any file using the Lightning lit:// path notation?** - - Yes, but only files for which the app has write permissions can be copied from Work to Work (apps don't run with root privileges). - -- **Q: Can I access the Lightning Storage in my UI (StreamLit, Web, ...)?** - - This is currently not supported but will be in the future. - -- **Q: Should I define my lit:// path in the __init__ or the run method?** - - You can declare a Lightning path anywhere you'd like. However, the ``.get()`` and ``.exists_*()`` methods only work inside of the run method of a LightningWork. - -- **Q:How often does Lightning synchronize the files between my Work?** - - Lightning does not synchronize the files between works. It only transfers the files once when the Work ``run`` method starts. - But you can call ``Path.get()`` as many times as you wish to transfer the latest file into the current Work. - -- **Does Lightning provide me direct access to the shared cloud folder?** - - No, and this is on purpose. This restriction forces developers to build modular components that can be shared and integrated - into apps easily. This would be much harder to achieve if file paths in these components would reference a global shared storage. - ----- - -.. raw:: html - -
-
- -.. displayitem:: - :header: Learn about the Drive Object. - :description: Put, List and Get Files From a Shared Drive Disk. - :col_css: col-md-4 - :button_link: drive.html - :height: 180 - :tag: Basic - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/storage/storage.rst b/docs/source-app/glossary/storage/storage.rst deleted file mode 100644 index af115a813fc80..0000000000000 --- a/docs/source-app/glossary/storage/storage.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. _storage: - -####### -Storage -####### - -**Audience:** Users who want to share files between components. - - -The Lightning Storage system makes it easy to share files between LightningWork so you can run your app both locally and in the cloud without changing the code. - - -Lightning storage provides two solutions :class:`~lightning.app.storage.drive.Drive` and :class:`~lightning.app.storage.path.Path` to deal with files locally and in the cloud likewise. - - ----- - -.. raw:: html - -
-
- -.. displayitem:: - :header: Learn about the differences between Drive vs Path. - :description: Learn about their differences. - :col_css: col-md-4 - :button_link: differences.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: The Drive Object. - :description: Put, List and Get Files From a Shared Drive Disk. - :col_css: col-md-4 - :button_link: drive.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: The Path Object. - :description: Transfer Files From One Component to Another by Reference. - :col_css: col-md-4 - :button_link: path.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
- - ----- - -******** -Examples -******** - - - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a File Server - :description: Learn how to use Drive to upload / download files to your app. - :col_css: col-md-4 - :button_link: ../../examples/file_server/file_server.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/use_local_lightning.rst b/docs/source-app/glossary/use_local_lightning.rst deleted file mode 100644 index 1efc8e730e97a..0000000000000 --- a/docs/source-app/glossary/use_local_lightning.rst +++ /dev/null @@ -1,15 +0,0 @@ -################################################################ -How to run an app on the cloud with a local version of lightning -################################################################ - -The lightning cloud uses the latest release by default. However, you might want to run your app with some local changes you've made to the lightning framework. To use your local version of lightning on the cloud, set the following environment variable: - -```bash -git clone https://github.com/Lightning-AI/lightning.git -cd lightning -pip install -e . -export PACKAGE_LIGHTNING=1 # <- this is the magic to use your version (not mainstream PyPI)! -lightning_app run app app.py --cloud -``` - -By setting `PACKAGE_LIGHTNING=1`, lightning packages the lightning source code in your local directory in addition to your app source code and uploads them to the cloud. diff --git a/docs/source-app/index.rst b/docs/source-app/index.rst deleted file mode 100644 index fb4258ad28e3a..0000000000000 --- a/docs/source-app/index.rst +++ /dev/null @@ -1,153 +0,0 @@ -.. lightning documentation master file, created by - sphinx-quickstart on Sat Sep 19 16:37:02 2020. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -####################### -Welcome to ⚡ Lightning -####################### -Build models, ML components and full stack AI apps ⚡ *Lightning fast*. - -**Featured examples of what you can do with Lightning:** - -| - -.. raw:: html - -
-
- -.. app_card:: - :title: Develop and Train - :description: Train a model (32 GPUs) - :width: 280 - :image: https://lightning-ai-docs.s3.amazonaws.com/develop_n_train_v1.jpg - :target: levels/basic/real_lightning_component_implementations.html#ex-pytorch-lightning-trainer - :preview: levels/basic/real_lightning_component_implementations.html#ex-pytorch-lightning-trainer - :tags: Training - -.. app_card:: - :title: Serve and deploy - :description: Develop a Model Server - :width: 280 - :image: https://lightning-ai-docs.s3.amazonaws.com/serve_n_deploy_v1.jpg - :target: examples/model_server_app/model_server_app.html - :preview: examples/model_server_app/model_server_app.html - :tags: Serving - -.. app_card:: - :title: Scale and build a product - :description: Production-ready generative AI app - :width: 280 - :app_id: HvUwbEG90E - :image: https://lightning-ai-docs.s3.amazonaws.com/scale_n_build_v1.jpg - :target: https://lightning.ai/app/HvUwbEG90E-Muse - :tags: AI App - -.. raw:: html - -
-
- ----- - -******************************** -Build self-contained, components -******************************** -Use Lightning, the hyper-minimalistic framework, to build machine learning components that can plug into existing ML workflows. -A Lightning component organizes arbitrary code to run on the cloud, manage its own infrastructure, cloud costs, networking, and more. -Focus on component logic and not engineering. - -Use components on their own, or compose them into full-stack AI apps with our next-generation Lightning orchestrator. - -.. raw:: html - -
- -
- -| - -| - -**Run an example component on the cloud**: - -.. include:: ./levels/basic/hero_components.rst - -| - -Components run the same on the cloud and locally on your choice of hardware. - -.. lit_tabs:: - :code_files: landing_app_run.bash - :highlights: 5 - :height: 150px - :code_only: True - -Explore pre-built community components in `our gallery `_. - -| - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Get started - :description: Learn to build Lightning components step-by-step. - :col_css: col-md-12 - :button_link: levels/basic/index.html - :height: 160 - :tag: 10 minutes - -.. raw:: html - -
-
- -.. raw:: html - -
- -.. toctree:: - :maxdepth: 1 - :caption: Home - - self - Install - -.. toctree:: - :maxdepth: 1 - :caption: Get started in steps - - Basic - Intermediate - Advanced - -.. toctree:: - :maxdepth: 1 - :caption: Core API Reference - - LightningApp - LightningFlow - LightningWork - -.. toctree:: - :maxdepth: 1 - :caption: Addons API Reference - - api_reference/components - api_reference/frontend - api_reference/runners - api_reference/storage - -.. toctree:: - :maxdepth: 1 - :caption: More - - Examples - Glossary - How-to diff --git a/docs/source-app/install/install_beginner.rst b/docs/source-app/install/install_beginner.rst deleted file mode 100644 index f690ef74e7d2f..0000000000000 --- a/docs/source-app/install/install_beginner.rst +++ /dev/null @@ -1,117 +0,0 @@ -:orphan: - -.. _install_beginner: - -############################# -What is a virtual environment -############################# -A virtual environment keeps the packages you install isolated from the rest of your system. -This allows you to work on multiple projects that have different and potentially conflicting requirements, and it -keeps your system Python installation clean. - -.. raw:: html - - - ----- - -We will describe two choices here, pick one: - - -1. :ref:`Python virtualenv `. -2. :ref:`Conda virtual environment `. - ----- - -.. _python-virtualenv: - -******************** -1. Python Virtualenv -******************** - -First, make sure that you have Python 3.8+ installed on your system. - -.. code-block:: bash - - python3 --version - -If you can't run the command above or it returns a version older than 3.8, -`install the latest version of Python `_. -After installing it, make sure you can run the above command without errors. - ----- - -Creating a Virtual Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When starting with a new Python project, you typically want to create a new Python virtual environment. -Navigate to the location of your project and run the following command: - -.. code-block:: bash - - python3 -m venv lightning - -The name of the environment here is *lightning* but you can choose any other name you like. -By running the above command, Python will create a new folder *lightning* in the current working directory. - ----- - -Activating the Virtual Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Before you can install packages into the environment, you need to activate it: - -.. code-block:: bash - - source lightning/bin/activate - -You need to do this step every time you want to work on your project / open the terminal. -With your virtual environment activated, you are now ready to -:doc:`install Lightning ` and get started with Apps! - ----- - -.. _conda: - -******** -2. Conda -******** - -To get started, you first need to download and install the `Miniconda package manager `_. -To check that the installation was successful, open an new terminal and run: - -.. code:: bash - - conda - -It should return a list of commands. - ----- - -Creating a Conda Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When starting with a new Python project, you typically want to create a new Conda virtual environment. -Navigate to the location of your project and run the following command: - -.. code-block:: bash - - conda create --yes --name lightning python=3.8 - -The name of the environment here is *lightning* but you can choose any other name you like. -Note how we can also specify the Python version here. - ----- - -Activating the Conda Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Before you can install packages into the environment, you need to activate it: - -.. code-block:: bash - - conda activate lightning - -You need to do this step every time you want to work on your project / open the terminal. -With your virtual environment activated, you are now ready to -:doc:`install Lightning ` and get started with Apps! diff --git a/docs/source-app/install/installation.rst b/docs/source-app/install/installation.rst deleted file mode 100644 index 294e26853007d..0000000000000 --- a/docs/source-app/install/installation.rst +++ /dev/null @@ -1,29 +0,0 @@ - -.. _install: - - -############ -Installation -############ - -**Prerequisites**: Use Python 3.8.x or later (3.8.x, 3.9.x, 3.10.x). We also recommend you install in a virtual environment (learn how). - -.. lit_tabs:: - :descriptions: Pip; Macs, Apple Silicon (M1/M2/M3); Windows - :code_files: pip.bash; mac.bash; windows.bash - :tab_rows: 4 - :height: 180px - ----- - -************ -Troubleshoot -************ -If you encounter issues during installation join our community discord and share the output of the following command: - -.. code:: bash - - pip list | grep lightning - -.. join_slack:: - :align: left diff --git a/docs/source-app/install/mac.bash b/docs/source-app/install/mac.bash deleted file mode 100644 index 22825bb246530..0000000000000 --- a/docs/source-app/install/mac.bash +++ /dev/null @@ -1,5 +0,0 @@ -# needed for M1/M2/M3 -export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 -export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 - -pip install lightning diff --git a/docs/source-app/install/pip.bash b/docs/source-app/install/pip.bash deleted file mode 100644 index f6d38b76b8107..0000000000000 --- a/docs/source-app/install/pip.bash +++ /dev/null @@ -1 +0,0 @@ -pip install lightning diff --git a/docs/source-app/install/windows.bash b/docs/source-app/install/windows.bash deleted file mode 100644 index 150b04e5b4d5d..0000000000000 --- a/docs/source-app/install/windows.bash +++ /dev/null @@ -1,4 +0,0 @@ -# install pip -# install git -# setup an alias for Python: python=python3 -# Add the root folder of Lightning to the Environment Variables to PATH diff --git a/docs/source-app/intro.rst b/docs/source-app/intro.rst deleted file mode 100644 index c975ee7090bbe..0000000000000 --- a/docs/source-app/intro.rst +++ /dev/null @@ -1,88 +0,0 @@ -:orphan: - -.. _what: - -################### -What is Lightning? -################### - -Lightning is a free, modular, distributed, and open-source framework for building -AI applications where the components you want to use interact together. - -Lightning apps can be built for **any AI use case**, ranging from AI research to -production-ready pipelines (and everything in between!). - -By abstracting the engineering boilerplate, Lightning allows researchers, data scientists, and software engineers to -build highly-scalable, production-ready AI apps using the tools and technologies of their choice, -regardless of their level of engineering expertise. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/Lightning.gif - :alt: What is Lightning gif. - :width: 100 % - ----- - -.. _why: - -*************** -Why Lightning? -*************** - - -Easy to learn -^^^^^^^^^^^^^ - -Lightning was built for creating AI apps, not for dev-ops. It offers an intuitive, pythonic -and highly composable interface that allows you to focus on solving the problems that are important to you. - ----- - -Quick to deliver -^^^^^^^^^^^^^^^^ - -Lightning speeds the development process by offering testable templates you can build from, -accelerating the process of moving from idea to prototype and finally to market. - ----- - -Easy to scale -^^^^^^^^^^^^^ - -Lightning provides a mirrored experience locally and in the cloud. The `lightning.ai `_. -cloud platform abstracts the infrastructure, so you can run your apps at any scale. - ----- - -Easy to collaborate -^^^^^^^^^^^^^^^^^^^ - -Lightning was built for collaboration. -By following the best MLOps practices provided through our documentation and example use cases, -you can deploy state-of-the-art ML applications that are ready to be used by teams of all sizes. - ----- - -***************************** -What's Novel With Lightning? -***************************** - - -Cloud Infra Made Simple and Pythonic -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Lightning is for building reactive, scalable, cost effective, easy-to-maintain and reliable ML products in the cloud without worrying about infrastructure. Lightning provides several engineering novelties to enable this: - -#. **Reactivity**: Lightning allows you to run stateful components distributed across different machines, so you can design async, dynamic and reactive workflows in python, without having to define DAGs. - -#. **Scalable & Cost-Effective**: Lightning provides a granular and simple way to run components preemptively or on-demand and on any desired resource such as CPU or GPU. It also enables you to easily transfer artifacts from one machine to another. - -#. **Reliability**: - - #. **Checkpointing**: Lightning apps can be paused and resumed from generated state and artifact-based checkpoints. - #. **Resilience**: Lightning has a strong fault-tolerance foundation. Your application can be written and tested to be resilient for cloud hazards at the component level. - #. **Testing Tools**: Lightning provides you with tools and best practices you can use to develop and test your application. All of our built-in templates have unit integration and end-to-end tests. - -#. **Easy to maintain**: - - #. **Easy Debugging**: Lightning apps can be debugged locally and in the cloud with **breakpoints** in any components. - #. **Non-Invasive**: Lightning is the glue that connects all parts of your workflow, but this is done in a non-invasive way by formalizing API contracts between components. In other words, your application can run someone else's code with little assumption. diff --git a/docs/source-app/landing_app.py b/docs/source-app/landing_app.py deleted file mode 100644 index fa9429a1b53cd..0000000000000 --- a/docs/source-app/landing_app.py +++ /dev/null @@ -1,12 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - -# run on a cloud machine ("cpu", "gpu", ...) -component = YourComponent(cloud_compute=CloudCompute("cpu")) -app = LightningApp(component) diff --git a/docs/source-app/landing_app_run.bash b/docs/source-app/landing_app_run.bash deleted file mode 100644 index e4bed41dadddd..0000000000000 --- a/docs/source-app/landing_app_run.bash +++ /dev/null @@ -1,5 +0,0 @@ -# install lightning -pip install lightning - -# run the app on the --cloud (--setup installs deps automatically) -lightning_app run app app.py --setup --cloud diff --git a/docs/source-app/levels/advanced/index.rst b/docs/source-app/levels/advanced/index.rst deleted file mode 100644 index 4ba7d09d68ba7..0000000000000 --- a/docs/source-app/levels/advanced/index.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. _advanced_level: - -.. toctree:: - :maxdepth: 1 - :hidden: - - start_dynamic_components - level_16 - level_17 - level_18 - level_19 - level_20 - -############### -Advanced skills -############### -Learn to build nested components with advanced functionality. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 9: Start dynamic components - :description: Learn to start works dynamically - :button_link: start_dynamic_components.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 10: Check component status - :description: Learn to use work status to coordinate complex apps. - :button_link: level_16.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level: Nest flows - :description: Learn to nest flows into other flows. - :button_link: level_14.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level: Develop reactive apps. - :description: Learn to develop reactive Lightning Apps. Lightning shines with reactive workflows. - :button_link: level_14.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level: Enable CLI commands for your app - :description: Speak to your app from a CLI over the network - :button_link: level_17.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level 11: Connect two components over the network - :description: Connect two LightningWorks over the network. - :button_link: level_14.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 13: Rerun components - :description: Learn to reuse components by passing different variables. - :button_link: level_17.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level 14: Handle Lightning App exceptions - :description: Learn to handle Lightning App exceptions. - :button_link: level_19.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/advanced/level_16.rst b/docs/source-app/levels/advanced/level_16.rst deleted file mode 100644 index 58c1e1fc7e0ad..0000000000000 --- a/docs/source-app/levels/advanced/level_16.rst +++ /dev/null @@ -1,10 +0,0 @@ -########################### -Level 16: Check Work status -########################### -**Audience:** Users who want to stop/start Lightning Work based on a status. - -**Prereqs:** Level 16+ - ----- - -.. include:: ../../core_api/lightning_work/status_content.rst diff --git a/docs/source-app/levels/advanced/level_17.rst b/docs/source-app/levels/advanced/level_17.rst deleted file mode 100644 index 6650860eaa177..0000000000000 --- a/docs/source-app/levels/advanced/level_17.rst +++ /dev/null @@ -1,10 +0,0 @@ -########################## -Level 17: Rerun components -########################## -**Audience:** Users who want Work.run() to activate multiple times in an app. - -**Prereqs:** Level 16+ and read the :doc:`Event Loop guide <../../glossary/event_loop>`. - ----- - -.. include:: ../../workflows/run_work_once_content.rst diff --git a/docs/source-app/levels/advanced/level_18.rst b/docs/source-app/levels/advanced/level_18.rst deleted file mode 100644 index 87fba3eb8bcc9..0000000000000 --- a/docs/source-app/levels/advanced/level_18.rst +++ /dev/null @@ -1,10 +0,0 @@ -############################################## -Level 18: Share objects between LightningWorks -############################################## -**Audience:** Users moving DataFrames or outputs, between Lightning Works (usually data engineers). - -**Prereqs:** Level 16+ and know about the Pandas library and read the :doc:`Access app state guide <../../workflows/access_app_state>`. - ----- - -.. include:: ../../core_api/lightning_work/payload_content.rst diff --git a/docs/source-app/levels/advanced/level_19.rst b/docs/source-app/levels/advanced/level_19.rst deleted file mode 100644 index 99a859e1ad2ca..0000000000000 --- a/docs/source-app/levels/advanced/level_19.rst +++ /dev/null @@ -1,11 +0,0 @@ -######################################### -Level 19: Handle Lightning App exceptions -######################################### - -**Audience:** Users who want to make Lightning Apps more robust to potential issues. - -**Prereqs:** Level 16+ - ----- - -.. include:: ../../core_api/lightning_work/handling_app_exception_content.rst diff --git a/docs/source-app/levels/advanced/level_20.rst b/docs/source-app/levels/advanced/level_20.rst deleted file mode 100644 index 1d045e85fc692..0000000000000 --- a/docs/source-app/levels/advanced/level_20.rst +++ /dev/null @@ -1,11 +0,0 @@ -####################################### -Level 20: Enable dynamic LightningWorks -####################################### - -**Audience:** Users who want to create/run/stop multiple LightningWorks not defined at app instantiation. - -**Prereqs:** Level 16+ - ----- - -.. include:: ../../core_api/lightning_app/dynamic_work_content.rst diff --git a/docs/source-app/levels/advanced/start_dynamic_components.rst b/docs/source-app/levels/advanced/start_dynamic_components.rst deleted file mode 100644 index 2e91bc2f632f0..0000000000000 --- a/docs/source-app/levels/advanced/start_dynamic_components.rst +++ /dev/null @@ -1,38 +0,0 @@ -############################### -Level: Start dynamic components -############################### -**Audience:** Users who want to run a Lightning Component in parallel (asynchronously). - -**Prereqs:** You must have finished the :doc:`Basic levels <../basic/index>`. - ----- - -.. include:: ../../workflows/run_work_in_parallel_content.rst - ----- - -********************************************** -Next steps: Share variables between components -********************************************** -Now that you know how to run components in parallel, we'll learn to share variables -across components to simplify complex workflows. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 6: Share variables between components - :description: Learn to connect components - :col_css: col-md-12 - :button_link: share_variables_between_lightning_components.html - :height: 150 - :tag: 10 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/basic/build_a_dag.rst b/docs/source-app/levels/basic/build_a_dag.rst deleted file mode 100644 index e430306447797..0000000000000 --- a/docs/source-app/levels/basic/build_a_dag.rst +++ /dev/null @@ -1,20 +0,0 @@ -:orphan: - -########################### -Example: Deploy a model API -########################### - -**Prereqs:** You have an app already running locally. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming soon! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. diff --git a/docs/source-app/levels/basic/build_a_lightning_component.rst b/docs/source-app/levels/basic/build_a_lightning_component.rst deleted file mode 100644 index 00b777e150813..0000000000000 --- a/docs/source-app/levels/basic/build_a_lightning_component.rst +++ /dev/null @@ -1,154 +0,0 @@ -############################################## -Level 1: Package code in a lightning component -############################################## - -**Prereqs:** You know *basic* Python. - -**Goal:** In this guide you'll learn to develop `a Lightning component `_. - - -********************************* -Why you need Lightning components -********************************* -A Lightning component is a self-contained, modular machine-learning component -that you can plug into your existing ML workflows. A Lightning component organizes arbitrary code so it can run on the cloud, manages -its own infrastructure, cloud costs, networking and more. Connect components using your current workflow management tools or -our :doc:`next-generation reactive orchestrator <../intermediate/index>`. - -Components run on the cloud or your laptop without code changes 🤯🤯. - -.. raw:: html - -
- -
- -| - -Organizing your code into Lightning components offers these benefits: - -.. collapse:: Build systems not scripts - - | - - The Lightning structure forces best practices so you don't have to be an expert production engineer. - Although it feels like you're writing a script, you are actually building a production-ready system. - -.. collapse:: Cost control - - | - - The component run-time has been optimized for cost management to support the largest machine-learning workloads. - Lower your cloud bill with machines that shut down or spin up faster. - -.. collapse:: For beginners: Code like an expert - - | - - Lightning embeds the best practices of building production-ready full stack AI apps into your - coding experience. You can write code like you normally do, and the Lightning structure - ensures your code is implicitly production ready... even if you're just doing research. - -.. collapse:: For experts: Scale with full control - - | - - if you know what you are doing, Lightning gives you full control to manage your own - scaling logic, fault-tolerance and even pre-provisioning, all from Python. - -.. collapse:: Integrate into your current workflow tools - - | - - Lightning components are self-contained pieces of functionality. Add them to your current workflow - tools to quickly fill in gaps in your ML workflow such as monitoring drift, training LLMs and more. - You can (optionally) use the Lightning App to integrate components into a cohesive workflow. - -.. collapse:: Packaged code - - | - - Lightning apps bundles components into an app that runs in any environment. The same code will run on your laptop, - or any cloud or private clusters. You don't have to think about the cluster or know anything about the cloud. - -.. collapse:: Rapid iteration - - | - - Iterate through ideas in hours not months because you don't have to learn a million other concepts that the components - handle for you such as kubernetes, cost management, auto-scaling and more. - -.. collapse:: Modularity - - | - - Components are modular and inter-operable by design. Leverage our vibrant community of components so you don't - have to build each piece of the system yourself. - ----- - -***************** -Install Lightning -***************** -First, install Lightning. - -.. lit_tabs:: - :descriptions: Pip; Macs, Apple Silicon (M1/M2/M3); Windows - :code_files: /install/pip.bash; /install/mac.bash; /install/windows.bash - :tab_rows: 4 - :height: 180px - ----- - -************************** -Build your first component -************************** -A Lightning component organizes arbitrary code so it can run on the cloud, manages its own infrastructure, cloud costs, networking and more - -**Run one of these components!** - -.. include:: ./hero_components.rst - ----- - -************ -Key features -************ -You now know enough to build a self-contained component that runs any Python code on the cloud that can be connected to form a -powerful Lightning app. Here are a few key features available to super-charge your work: - -.. lit_tabs:: - :titles: 15+ accelerators; Auto-stop idle machines; Auto-timeout submitted work; Use spot machines (~70% discount); Work with massive datasets; Mount cloud storage; Use a custom container - :code_files: ./key_features/accelerators.py; ./key_features/idle_machine.py; ./key_features/auto_timeout.py; ./key_features/spot.py; ./key_features/massive_dataset.py; ./key_features/mount_data.py; ./key_features/custom_container.py; - :highlights: 11;11;11;11;11;2,7,10, 11; 11 - :enable_run: true - :tab_rows: 3 - :height: 430px - ----- - -******************************************** -Next: Explore real component implementations -******************************************** -In this section we introduced components. Let's explore -real component implementations in-depth. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 2: Explore real component implementations - :description: Go deep into real component implementations. - :col_css: col-md-12 - :button_link: real_lightning_component_implementations.html - :height: 150 - :tag: beginner - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/basic/create_a_model_demo.rst b/docs/source-app/levels/basic/create_a_model_demo.rst deleted file mode 100644 index 72750717efde4..0000000000000 --- a/docs/source-app/levels/basic/create_a_model_demo.rst +++ /dev/null @@ -1,20 +0,0 @@ -:orphan: - -############################ -Example: Create a model demo -############################ - -**Prereqs:** You have an app already running locally. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming soon! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. diff --git a/docs/source-app/levels/basic/deploy_ai_model_api.rst b/docs/source-app/levels/basic/deploy_ai_model_api.rst deleted file mode 100644 index e430306447797..0000000000000 --- a/docs/source-app/levels/basic/deploy_ai_model_api.rst +++ /dev/null @@ -1,20 +0,0 @@ -:orphan: - -########################### -Example: Deploy a model API -########################### - -**Prereqs:** You have an app already running locally. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming soon! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. diff --git a/docs/source-app/levels/basic/hello_components/code_run_cloud.bash b/docs/source-app/levels/basic/hello_components/code_run_cloud.bash deleted file mode 100644 index f81431222724c..0000000000000 --- a/docs/source-app/levels/basic/hello_components/code_run_cloud.bash +++ /dev/null @@ -1 +0,0 @@ -lightning_app run app app.py --cloud diff --git a/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash b/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash deleted file mode 100644 index 09435ff5d7caa..0000000000000 --- a/docs/source-app/levels/basic/hello_components/code_run_cloud_setup.bash +++ /dev/null @@ -1 +0,0 @@ -lightning_app run app app.py --setup --cloud diff --git a/docs/source-app/levels/basic/hello_components/code_run_local.bash b/docs/source-app/levels/basic/hello_components/code_run_local.bash deleted file mode 100644 index 1f8c994a84b1d..0000000000000 --- a/docs/source-app/levels/basic/hello_components/code_run_local.bash +++ /dev/null @@ -1 +0,0 @@ -lightning_app run app app.py diff --git a/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash b/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash deleted file mode 100644 index 45b0529de3736..0000000000000 --- a/docs/source-app/levels/basic/hello_components/code_run_local_setup.bash +++ /dev/null @@ -1 +0,0 @@ -lightning_app run app app.py --setup diff --git a/docs/source-app/levels/basic/hello_components/deploy_model.py b/docs/source-app/levels/basic/hello_components/deploy_model.py deleted file mode 100644 index 7911f6a3158ac..0000000000000 --- a/docs/source-app/levels/basic/hello_components/deploy_model.py +++ /dev/null @@ -1,31 +0,0 @@ -# !pip install torchvision -from lightning.app import LightningApp, CloudCompute -from lightning.app.components.serve import PythonServer, Image, Number -import base64, io, torchvision, torch -from PIL import Image as PILImage - - -class PyTorchServer(PythonServer): - def setup(self): - self._model = torchvision.models.resnet18(pretrained=True) - self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self._model.to(self._device) - - def predict(self, request): - image = base64.b64decode(request.image.encode("utf-8")) - image = PILImage.open(io.BytesIO(image)) - transforms = torchvision.transforms.Compose([ - torchvision.transforms.Resize(224), - torchvision.transforms.ToTensor(), - torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) - ]) - image = transforms(image) - image = image.to(self._device) - prediction = self._model(image.unsqueeze(0)) - return {"prediction": prediction.argmax().item()} - - -component = PyTorchServer( - input_type=Image, output_type=Number, cloud_compute=CloudCompute('gpu') -) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/hello_world.py b/docs/source-app/levels/basic/hello_components/hello_world.py deleted file mode 100644 index a716fecb2fb2c..0000000000000 --- a/docs/source-app/levels/basic/hello_components/hello_world.py +++ /dev/null @@ -1,12 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - - -component = YourComponent() -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/hello_world_gpu.py b/docs/source-app/levels/basic/hello_components/hello_world_gpu.py deleted file mode 100644 index 67e5d92fb3666..0000000000000 --- a/docs/source-app/levels/basic/hello_components/hello_world_gpu.py +++ /dev/null @@ -1,12 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - -# run on a cloud machine ("cpu", "gpu", ...) -compute = CloudCompute("gpu") -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/multi_node.py b/docs/source-app/levels/basic/hello_components/multi_node.py deleted file mode 100644 index 03ce0fbec2341..0000000000000 --- a/docs/source-app/levels/basic/hello_components/multi_node.py +++ /dev/null @@ -1,29 +0,0 @@ -# !pip install torch -from lightning.app import LightningWork, LightningApp, CloudCompute -from lightning.app.components import MultiNode - - -class MultiNodeComponent(LightningWork): - def run( - self, - main_address: str, - main_port: int, - node_rank: int, - world_size: int, - ): - print(f"ADD YOUR DISTRIBUTED CODE: {main_address=} {main_port=} {node_rank=} {world_size=}") - print("supports ANY ML library") - - - - - - - - - - -# gpu-multi-fast has 4 GPUs x 8 nodes = 32 GPUs -component = MultiNodeComponent(cloud_compute=CloudCompute("gpu-multi-fast")) -component = MultiNode(component, nodes=8) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/pl_multinode.py b/docs/source-app/levels/basic/hello_components/pl_multinode.py deleted file mode 100644 index 09480da44eee6..0000000000000 --- a/docs/source-app/levels/basic/hello_components/pl_multinode.py +++ /dev/null @@ -1,20 +0,0 @@ -# app.py -from lightning import Trainer -from lightning.app import LightningWork, LightningApp, CloudCompute -from lightning.app.components import LightningTrainerMultiNode -from lightning.pytorch.demos.boring_classes import BoringModel - - -class LightningTrainerDistributed(LightningWork): - def run(self): - model = BoringModel() - trainer = Trainer(max_epochs=10, strategy="ddp") - trainer.fit(model) - -# 8 GPUs: (2 nodes of 4 x v100) -component = LightningTrainerMultiNode( - LightningTrainerDistributed, - num_nodes=4, - cloud_compute=CloudCompute("gpu-fast-multi"), # 4 x v100 -) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/pt_multinode.py b/docs/source-app/levels/basic/hello_components/pt_multinode.py deleted file mode 100644 index 569a7c99b4d0c..0000000000000 --- a/docs/source-app/levels/basic/hello_components/pt_multinode.py +++ /dev/null @@ -1,60 +0,0 @@ -# app.py -# ! pip install torch -from lightning.app import LightningWork, LightningApp, CloudCompute -from lightning.app.components import MultiNode -import torch -from torch.nn.parallel.distributed import DistributedDataParallel - - -def distributed_train(local_rank: int, main_address: str, main_port: int, num_nodes: int, node_rank: int, nprocs: int): - # 1. SET UP DISTRIBUTED ENVIRONMENT - global_rank = local_rank + node_rank * nprocs - world_size = num_nodes * nprocs - - if torch.distributed.is_available() and not torch.distributed.is_initialized(): - torch.distributed.init_process_group( - "nccl" if torch.cuda.is_available() else "gloo", - rank=global_rank, - world_size=world_size, - init_method=f"tcp://{main_address}:{main_port}", - ) - - # 2. PREPARE DISTRIBUTED MODEL - model = torch.nn.Linear(32, 2) - device = torch.device(f"cuda:{local_rank}") if torch.cuda.is_available() else torch.device("cpu") - model = DistributedDataParallel(model, device_ids=[local_rank] if torch.cuda.is_available() else None).to(device) - - # 3. SETUP LOSS AND OPTIMIZER - criterion = torch.nn.MSELoss() - optimizer = torch.optim.SGD(model.parameters(), lr=0.01) - - # 4.TRAIN THE MODEL FOR 50 STEPS - for step in range(50): - model.zero_grad() - x = torch.randn(64, 32).to(device) - output = model(x) - loss = criterion(output, torch.ones_like(output)) - print(f"global_rank: {global_rank} step: {step} loss: {loss}") - loss.backward() - optimizer.step() - - # 5. VERIFY ALL COPIES OF THE MODEL HAVE THE SAME WEIGTHS AT END OF TRAINING - weight = model.module.weight.clone() - torch.distributed.all_reduce(weight) - assert torch.equal(model.module.weight, weight / world_size) - - print("Multi Node Distributed Training Done!") - -class PyTorchDistributed(LightningWork): - def run(self, main_address: str, main_port: int, num_nodes: int, node_rank: int): - nprocs = torch.cuda.device_count() if torch.cuda.is_available() else 1 - torch.multiprocessing.spawn( - distributed_train, - args=(main_address, main_port, num_nodes, node_rank, nprocs), - nprocs=nprocs - ) - -# 32 GPUs: (8 nodes x 4 v 100) -compute = CloudCompute("gpu-fast-multi") # 4xV100 -component = MultiNode(PyTorchDistributed, num_nodes=8, cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/run_ptl_script.py b/docs/source-app/levels/basic/hello_components/run_ptl_script.py deleted file mode 100644 index 46501310e2aa7..0000000000000 --- a/docs/source-app/levels/basic/hello_components/run_ptl_script.py +++ /dev/null @@ -1,13 +0,0 @@ -# app.py -# !curl https://raw.githubusercontent.com/Lightning-AI/lightning/master/examples/app/multi_node/pl_boring_script.py -o pl_boring_script.py -from lightning.app import LightningApp, CloudCompute -from lightning.app.components.training import LightningTrainerScript - -# run script that trains PyTorch with the Lightning Trainer -model_script = 'pl_boring_script.py' -component = LightningTrainerScript( - model_script, - num_nodes=1, - cloud_compute=CloudCompute("gpu") -) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/hello_components/streamlit_demo.py b/docs/source-app/levels/basic/hello_components/streamlit_demo.py deleted file mode 100644 index 41ad3988d908d..0000000000000 --- a/docs/source-app/levels/basic/hello_components/streamlit_demo.py +++ /dev/null @@ -1,30 +0,0 @@ -# app.py -# !pip install streamlit omegaconf scipy -# !pip install torch -from lightning.app import LightningApp -import torch -from io import BytesIO -from functools import partial -from scipy.io.wavfile import write -import streamlit as st - - -class StreamlitApp(app.components.ServeStreamlit): - def build_model(self): - sample_rate = 48000 - model, _ = torch.hub.load('snakers4/silero-models', model='silero_tts',speaker="v3_en") - return partial(model.apply_tts, sample_rate=sample_rate, speaker="en_0"), sample_rate - - def render(self): - st.title("Text To Speech") - text = st.text_input("Text:", "Lightning Apps are the best!") - - if text: - model, sample_rate = self.model - audio_numpy = model(text).numpy() - audio = BytesIO() - write(audio, sample_rate, audio_numpy) - audio.seek(0) - st.audio(audio) - -app = LightningApp(StreamlitApp()) diff --git a/docs/source-app/levels/basic/hello_components/train_ptl.py b/docs/source-app/levels/basic/hello_components/train_ptl.py deleted file mode 100644 index 3fa9684f9213b..0000000000000 --- a/docs/source-app/levels/basic/hello_components/train_ptl.py +++ /dev/null @@ -1,15 +0,0 @@ -# A hello world component -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - - -# run on a cloud machine -compute = CloudCompute("cpu") -worker = YourComponent(cloud_compute=compute) -app = LightningApp(worker) diff --git a/docs/source-app/levels/basic/hello_components/train_pytorch.py b/docs/source-app/levels/basic/hello_components/train_pytorch.py deleted file mode 100644 index fa54c577bd19b..0000000000000 --- a/docs/source-app/levels/basic/hello_components/train_pytorch.py +++ /dev/null @@ -1,28 +0,0 @@ -# app.py -# ! pip install torch -from lightning.app import LightningWork, LightningApp, CloudCompute -import torch - -class PyTorchComponent(LightningWork): - def run(self): - device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - model = torch.nn.Sequential(torch.nn.Linear(1, 1), - torch.nn.ReLU(), - torch.nn.Linear(1, 1)) - model.to(device) - criterion = torch.nn.MSELoss() - optimizer = torch.optim.SGD(model.parameters(), lr=0.1) - - for step in range(10000): - model.zero_grad() - x = torch.tensor([0.8]).to(device) - target = torch.tensor([1.0]).to(device) - output = model(x) - loss = criterion(output, target) - print(f'step: {step}. loss {loss}') - loss.backward() - optimizer.step() - -compute = CloudCompute('gpu') -componet = PyTorchComponent(cloud_compute=compute) -app = LightningApp(componet) diff --git a/docs/source-app/levels/basic/hello_components/xgboost.py b/docs/source-app/levels/basic/hello_components/xgboost.py deleted file mode 100644 index 68cc2c181e050..0000000000000 --- a/docs/source-app/levels/basic/hello_components/xgboost.py +++ /dev/null @@ -1,21 +0,0 @@ -# app.py -# !pip install scikit-learn xgboost -from lightning.app import LightningWork, LightningApp -from sklearn import datasets -from sklearn.model_selection import train_test_split -from xgboost import XGBClassifier - -class XGBoostComponent(LightningWork): - def run(self): - iris = datasets.load_iris() - X, y = iris.data, iris.target - - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) - - bst = XGBClassifier(verbosity=3) - bst.fit(X_train, y_train) - preds = bst.predict(X_test) - print(f'preds: {preds}') - - -app = LightningApp(XGBoostComponent()) diff --git a/docs/source-app/levels/basic/hello_components/xgboost_gpu.py b/docs/source-app/levels/basic/hello_components/xgboost_gpu.py deleted file mode 100644 index f8058a3169e34..0000000000000 --- a/docs/source-app/levels/basic/hello_components/xgboost_gpu.py +++ /dev/null @@ -1,22 +0,0 @@ -# app.py -# !pip install sklearn xgboost -# !conda install py-xgboost-gpu -from lightning.app import LightningWork, LightningApp, CloudCompute -from sklearn import datasets -from sklearn.model_selection import train_test_split -from xgboost import XGBClassifier - -class XGBoostComponent(LightningWork): - def run(self): - iris = datasets.load_iris() - X, y = iris.data, iris.target - - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) - - bst = XGBClassifier(tree_method='gpu_hist', gpu_id=0, verbosity=3) - bst.fit(X_train, y_train) - preds = bst.predict(X_test) - print(f'preds: {preds}') - -compute = CloudCompute('gpu') -app = LightningApp(XGBoostComponent(cloud_compute=compute)) diff --git a/docs/source-app/levels/basic/hero_components.rst b/docs/source-app/levels/basic/hero_components.rst deleted file mode 100644 index da1d9d076a794..0000000000000 --- a/docs/source-app/levels/basic/hero_components.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. lit_tabs:: - :titles: Hello world; Hello GPU world; PyTorch & ⚡⚡⚡ Trainer (1+ cloud GPUs); Train PyTorch (cloud GPU); Train PyTorch (32 cloud GPUs); Deploy a model on cloud GPUs; Run a model script; XGBoost; Streamlit demo - :code_files: /levels/basic/hello_components/hello_world.py; /levels/basic/hello_components/hello_world_gpu.py; /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/train_pytorch.py; /levels/basic/hello_components/pt_multinode.py; /levels/basic/hello_components/deploy_model.py; /levels/basic/hello_components/run_ptl_script.py; /levels/basic/hello_components/xgboost.py; /levels/basic/hello_components/streamlit_demo.py - :highlights: 7; 10, 11; 9-11, 16, 17; 4, 8, 12, 18-19, 26; 5, 10, 22, 27, 31, 41, 57-59; 3, 11-12, 25, 29; 7, 10; 15, 21; 9, 15, 24 - :works: [{"name":"root.work","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"default","preemptible":false,"shmSize":0},"networkConfig":[{"name":"dzodf","port":61304}]}}];[{"name":"root.work","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"qnlgd","port":61516}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu","preemptible":false,"shmSize":0}}}];[{"name":"root.ws.0","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"ajfrc","port":61553}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.1","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"ttyqc","port":61554}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.2","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"svyej","port":61555}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.3","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"parme","port":61556}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}}];[{"name":"root.work","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"cutdu","port":61584}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu","preemptible":false,"shmSize":0}}}];[{"name":"root.ws.0","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"whhby","port":61613}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.1","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"yhjtf","port":61614}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.2","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"rqwkt","port":61615}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.3","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"pjdsj","port":61616}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.4","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"efdor","port":61617}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.5","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"pxmso","port":61618}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.6","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"feevy","port":61619}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}},{"name":"root.ws.7","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"tbmse","port":61620}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu-fast-multi","preemptible":false,"shmSize":0}}}];[{"name":"root.work","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"umqqg","port":7777}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"gpu","preemptible":false,"shmSize":0}}}];[];[{"name":"root.work","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"tggba","port":61729}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"default","preemptible":false,"shmSize":0}}}];[{"name":"root.work","spec":{"buildSpec":{"commands":[],"pythonDependencies":{"packageManager":"PACKAGE_MANAGER_PIP","packages":""}},"drives":[],"networkConfig":[{"name":"hpyaz","port":61763}],"userRequestedComputeConfig":{"count":1,"diskSize":0,"name":"default","preemptible":false,"shmSize":0}}}] - :enable_run: true - :tab_rows: 3 - :height: 620px diff --git a/docs/source-app/levels/basic/index.rst b/docs/source-app/levels/basic/index.rst deleted file mode 100644 index 2912b69b6b7ff..0000000000000 --- a/docs/source-app/levels/basic/index.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. _level_basic: - -.. toctree:: - :maxdepth: 1 - :hidden: - - build_a_lightning_component - real_lightning_component_implementations - save_money_on_cloud_costs - -############ -Basic skills -############ -Learn to package your code into Lightning components which can plug into your existing ML workflows. - -A Lightning component organizes arbitrary code so it can run on the cloud, manages -its own infrastructure, cloud costs, networking and more. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 1: Package code in a Lightning component - :description: Learn to package your code into Lightning components which can plug into your existing ML workflows. - :button_link: build_a_lightning_component.html - :col_css: col-md-6 - :height: 170 - :tag: 10 minutes - -.. displayitem:: - :header: Level 2: Explore real component implementations - :description: Go deep into real component implementations. - :button_link: real_lightning_component_implementations.html - :col_css: col-md-6 - :height: 170 - :tag: 10 minutes - -.. displayitem:: - :header: Level 3: Save money on cloud costs - :description: Explore key Lightning features that save you cloud costs and improve performance. - :button_link: save_money_on_cloud_costs.html - :col_css: col-md-6 - :height: 150 - :tag: 10 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/basic/key_features/accelerators.py b/docs/source-app/levels/basic/key_features/accelerators.py deleted file mode 100644 index fd27325fda74e..0000000000000 --- a/docs/source-app/levels/basic/key_features/accelerators.py +++ /dev/null @@ -1,21 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - -# custom accelerators -compute = CloudCompute('gpu') -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) - -# OTHER ACCELERATORS: -# compute = CloudCompute('default') # 1 CPU -# compute = CloudCompute('cpu-medium') # 8 CPUs -# compute = CloudCompute('gpu') # 1 T4 GPU -# compute = CloudCompute('gpu-fast-multi') # 4 V100 GPU -# compute = CloudCompute('p4d.24xlarge') # AWS instance name (8 A100 GPU) -# compute = ... diff --git a/docs/source-app/levels/basic/key_features/auto_timeout.py b/docs/source-app/levels/basic/key_features/auto_timeout.py deleted file mode 100644 index 73d6281b602d5..0000000000000 --- a/docs/source-app/levels/basic/key_features/auto_timeout.py +++ /dev/null @@ -1,13 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - -# if the machine hasn't started after 60 seconds, cancel the work -compute = CloudCompute('gpu', wait_timeout=60) -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/key_features/custom_container.py b/docs/source-app/levels/basic/key_features/custom_container.py deleted file mode 100644 index 5e574d43d90a8..0000000000000 --- a/docs/source-app/levels/basic/key_features/custom_container.py +++ /dev/null @@ -1,13 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - -# custom image (from any provider) -config= BuildConfig(image="gcr.io/google-samples/hello-app:1.0") -component = YourComponent(cloud_build_config=config) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/key_features/idle_machine.py b/docs/source-app/levels/basic/key_features/idle_machine.py deleted file mode 100644 index 89ab43355c8e6..0000000000000 --- a/docs/source-app/levels/basic/key_features/idle_machine.py +++ /dev/null @@ -1,13 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - -# stop the machine when idle for 10 seconds -compute = CloudCompute('gpu', idle_timeout=10) -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/key_features/massive_dataset.py b/docs/source-app/levels/basic/key_features/massive_dataset.py deleted file mode 100644 index 2c12b9cb4e8f7..0000000000000 --- a/docs/source-app/levels/basic/key_features/massive_dataset.py +++ /dev/null @@ -1,13 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - - -# use 100 GB of space on that machine (max size: 64 TB) -compute = CloudCompute('gpu', disk_size=100) -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/key_features/mount_data.py b/docs/source-app/levels/basic/key_features/mount_data.py deleted file mode 100644 index e6096d7e925d2..0000000000000 --- a/docs/source-app/levels/basic/key_features/mount_data.py +++ /dev/null @@ -1,13 +0,0 @@ -from lightning.app import LightningWork, LightningApp, CloudCompute -import os - - -class YourComponent(LightningWork): - def run(self): - os.listdir('/foo') - -# mount the files on the s3 bucket under this path -mount = Mount(source="s3://lightning-example-public/", mount_path="/foo") -compute = CloudCompute(mounts=mount) -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/key_features/spot.py b/docs/source-app/levels/basic/key_features/spot.py deleted file mode 100644 index b1a0291eeacee..0000000000000 --- a/docs/source-app/levels/basic/key_features/spot.py +++ /dev/null @@ -1,13 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningApp, CloudCompute - - -class YourComponent(LightningWork): - def run(self): - print('RUN ANY PYTHON CODE HERE') - -# spot machines can be turned off without notice, use for non-critical, resumable work -# request a spot machine, after 60 seconds of waiting switch to full-price -compute = CloudCompute('gpu', wait_timeout=60, spot=True) -component = YourComponent(cloud_compute=compute) -app = LightningApp(component) diff --git a/docs/source-app/levels/basic/real_lightning_component_implementations.rst b/docs/source-app/levels/basic/real_lightning_component_implementations.rst deleted file mode 100644 index 391bda9ada605..0000000000000 --- a/docs/source-app/levels/basic/real_lightning_component_implementations.rst +++ /dev/null @@ -1,75 +0,0 @@ -############################################### -Level 2: Explore real component implementations -############################################### -**Audience:** Users who want to deeply understand what is possible with Lightning components. - -**Prereqs:** You must have finished :doc:`level 1 <../basic/build_a_lightning_component>`. - ----- - -*********************** -Real component examples -*********************** -Use this guide to understand what is happening in each type of component. -These are a few prototypical components. Since each component organizes -Python, you can build virtually infinite components for any use-case -you can think of. - ----- - -******************************* -Ex: PyTorch + Lightning Trainer -******************************* -This example shows how to train PyTorch with the Lightning trainer on your machine -or cloud GPUs without code changes. - -.. lit_tabs:: - :descriptions: import Lightning; We're using a demo LightningModule; Move your training code here (usually your main.py); Pass your component to the multi-node executor (it works on CPU or single GPUs also); Select the number of machines (nodes). Here we choose 4.; Choose from over 15+ machine types. This one has 4 v100 GPUs.; Initialize the App object that executes the component logic. - :code_files: /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/pl_multinode.py; /levels/basic/hello_components/pl_multinode.py; - :highlights: 2; 4; 9-11; 14-17; 16; 17; 19 - :enable_run: true - :tab_rows: 5 - :height: 420px - ----- - -********************************* -Ex: Deploy a PyTorch API endpoint -********************************* -This example shows how to deploy PyTorch and create an API - -.. lit_tabs:: - :descriptions: Shortcut to list dependencies without a requirements.txt file.; Import one of our serving components (high-performance ones are available on the enterprise tiers); Define the setup function to load your favorite pretrained models and do any kind of pre-processing.; Define the predict function which is called when the endpoint is hit.; Initialize the server and define the type of cloud machine to use. - :code_files: /levels/basic/hello_components/deploy_model.py; /levels/basic/hello_components/deploy_model.py; /levels/basic/hello_components/deploy_model.py; /levels/basic/hello_components/deploy_model.py; /levels/basic/hello_components/deploy_model.py; - :highlights: 1; 3; 10-12; 15-25; 28-30 - :enable_run: true - :tab_rows: 4 - :height: 620px - ----- - -************************* -Next: Save on cloud costs -************************* -Let's review key lightning features to help you run components more efficiently on the cloud -so you can save on cloud costs. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 3: Save money on cloud costs - :description: Explore key Lightning features that save you cloud costs and improve performance. - :button_link: save_money_on_cloud_costs.html - :col_css: col-md-12 - :height: 150 - :tag: 10 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/basic/run_jupyter_notebook_on_the_cloud.rst b/docs/source-app/levels/basic/run_jupyter_notebook_on_the_cloud.rst deleted file mode 100644 index 9bc5cdd5207ef..0000000000000 --- a/docs/source-app/levels/basic/run_jupyter_notebook_on_the_cloud.rst +++ /dev/null @@ -1,20 +0,0 @@ -:orphan: - -############################################# -Example: Develop a Jupyter Notebook component -############################################# - -**Prereqs:** You have an app already running locally. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming soon! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. diff --git a/docs/source-app/levels/basic/save_money_on_cloud_costs.rst b/docs/source-app/levels/basic/save_money_on_cloud_costs.rst deleted file mode 100644 index b2ff007ca8bba..0000000000000 --- a/docs/source-app/levels/basic/save_money_on_cloud_costs.rst +++ /dev/null @@ -1,51 +0,0 @@ -################################## -Level 3: Save money on cloud costs -################################## -**Audience:** Users who want to use the AWS cloud efficiently. - -**Prereqs:** You must have finished :doc:`level 1 <../basic/build_a_lightning_component>`. - ----- - -*********************************** -Save money with these optimizations -*********************************** -A Lightning component gives you fine-grain control over the cloud lifecycle of that component. - -Here are a few features that will enable you save a lot on your cloud costs: - -.. lit_tabs:: - :titles: 15+ accelerators; Auto-stop idle machines; Auto-timeout submitted work; Use spot machines (~70% discount); Work with massive datasets; Mount cloud storage; Use a custom container - :code_files: ./key_features/accelerators.py; ./key_features/idle_machine.py; ./key_features/auto_timeout.py; ./key_features/spot.py; ./key_features/massive_dataset.py; ./key_features/mount_data.py; ./key_features/custom_container.py; - :highlights: 11;11;11;11;11;1,7, 10, 11; 11 - :enable_run: true - :tab_rows: 3 - :height: 430px - ----- - -****************************** -Next: Coordinate 2+ components -****************************** -Now that you know how to organize arbitrary code inside a Lightning component, -learn to coordinate 2 or more components into workflows which we call Lightning apps. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Intermediate skills - :description: Learn to coordinate 2+ components into workflows which we call Lightning apps. - :button_link: ../intermediate/index.html - :col_css: col-md-12 - :height: 170 - :tag: 15 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/basic/scripts/toy_app_1_component.py b/docs/source-app/levels/basic/scripts/toy_app_1_component.py deleted file mode 100644 index e09df3ecb0d9c..0000000000000 --- a/docs/source-app/levels/basic/scripts/toy_app_1_component.py +++ /dev/null @@ -1,17 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp - -class Component(LightningWork): - def run(self, x): - print(x) - - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component = Component() - - def run(self): - self.component.run('i love Lightning') - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/basic/scripts/toy_app_1_component_pdb.py b/docs/source-app/levels/basic/scripts/toy_app_1_component_pdb.py deleted file mode 100644 index 6348f9c4ed46e..0000000000000 --- a/docs/source-app/levels/basic/scripts/toy_app_1_component_pdb.py +++ /dev/null @@ -1,18 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.pdb import set_trace - -class Component(LightningWork): - def run(self, x): - print(x) - set_trace() - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component = Component() - - def run(self): - self.component.run('i love Lightning') - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/basic/train_pytorch_on_the_cloud.rst b/docs/source-app/levels/basic/train_pytorch_on_the_cloud.rst deleted file mode 100644 index 1860094624f15..0000000000000 --- a/docs/source-app/levels/basic/train_pytorch_on_the_cloud.rst +++ /dev/null @@ -1,20 +0,0 @@ -:orphan: - -################################### -Example: Train PyTorch on the cloud -################################### - -**Prereqs:** You have an app already running locally. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming soon! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. diff --git a/docs/source-app/levels/expert/index.rst b/docs/source-app/levels/expert/index.rst deleted file mode 100644 index c98199b9432e7..0000000000000 --- a/docs/source-app/levels/expert/index.rst +++ /dev/null @@ -1,82 +0,0 @@ -:orphan: - -.. _expert_level: - -.. toctree:: - :maxdepth: 1 - :hidden: - -############# -Expert skills -############# - - -.. raw:: html - -
-
- -.. displayitem:: - :header: Level : Use custom containers - :description: Learn to use a custom cloud container. - :button_link: build_a_machine_learning_workflow.html - :col_css: col-md-6 - :height: 150 - :tag: basic - - -.. raw:: html - -
-
- ----- - -********************* -Intermediate Examples -********************* -As you work through the intermediate levels, try these examples: - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example: Develop a Github Repo Script Runner - :description: Develop a workflow to execute Github Repos - :button_link: ../../examples/github_repo_runner/github_repo_runner.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Example: Develop a file server - :description: Create a simple Lightning App (App) that allows users to upload files and list the uploaded files. - :button_link: ../../examples/file_server/file_server.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Example: Develop a Jupyter Notebook component - :description: Develop a LightningWork that runs a notebook on the cloud. - :button_link: run_jupyter_notebook_on_the_cloud.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Example: Create a model demo - :description: Demo POCs and MVPs which can be shared with a public web user interface. - :button_link: create_a_model_demo.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/connect_lightning_components.rst b/docs/source-app/levels/intermediate/connect_lightning_components.rst deleted file mode 100644 index 31f6cc1516441..0000000000000 --- a/docs/source-app/levels/intermediate/connect_lightning_components.rst +++ /dev/null @@ -1,116 +0,0 @@ -#################################################### -Level 4: Connect components into a full stack AI app -#################################################### - -**Audience:** Users who want to build apps with multiple components. - -**Prereqs:** You know how to :doc:`build a component <../basic/build_a_lightning_component>`. - ----- - -**************************** -What is a full stack AI app? -**************************** -In the ML world, workflows coordinate multiple pieces of code working together. In Lightning, -when we coordinate 2 or more :doc:`Lightning components <../basic/build_a_lightning_component>` working together, -we instead call it a Lightning App. The difference will become more obvious when we introduce reactive -workflows in the advanced section. - -For the time being, we'll go over how to coordinate 2 components together in a traditional workflow setting -and explain how it works. - -.. note:: If you've used workflow tools for Python, this page describes conventional DAGs. - In :doc:`level 6 `, we introduce reactive workflows that generalize beyond DAGs - so you can build complex systems without much effort. - ----- - -*********** -The toy app -*********** - -In this app, we define two components that run across 2 separate machines. One to train a model on a GPU machine and one to analyze the model -on a separate CPU machine. We save money by stopping the GPU machine when the work is done. - -.. lit_tabs:: - :titles: Import Lightning; Define Component 1; Define Component 2; Orchestrator; Connect component 1; Connect component 2; Implement run; Train; Analyze; Define app placeholder - :descriptions: First, import Lightning; This component trains a model on a GPU machine; This component analyzes a model on a CPU machine; Define the LightningFlow that orchestrates components; Component 1 will run on a CPU machine; Component 2 will run on an accelerated GPU machine; Describe the workflow in the run method; Training runs first and completes; Analyze runs after training completes; This allows the app to be runnable - :code_files: ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py - :highlights: 2; 5-7; 9-11; 13; 16; 17; 19; 20; 21; 23 - :enable_run: true - :tab_rows: 4 - :height: 460px - -| - -Now run the app: - -.. lit_tabs:: - :titles: Run on Lightning cloud; Your own hardware - :descriptions: Run to see these 2 components execute on separate machines 🤯; Run it locally without code changes 🤯🤯; - :code_files: ./level_2_scripts/code_run_cloud.bash; ./level_2_scripts/code_run_local.bash - :tab_rows: 7 - :height: 195px - -| - -Now you can develop distributed cloud apps on your laptop 🤯🤯🤯🤯! - ----- - -************* -Now you know: -************* - -Without going out of your way, you're now doing the following: (Hint: Click **visualize** to see an animation describing the code). - -.. lit_tabs:: - :titles: Orchestration; Distributed cloud computing; Multi-machine communication; Multi-machine communication; Multi-cloud; - :descriptions: Define orchestration in Python with full control-flow; The two pieces of independent Python code ran on separate machines 🤯🤯; The text "CPU machine 1" was sent from the flow machine to the machine running the TrainComponent; The text "GPU machine 2" was sent from the flow machine to the machine running the AnalyzeComponent; The full Lightning app can move across clusters and clouds - :code_files: ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; ./level_2_scripts/hello_app.py; - :tab_rows: 4 - :highlights: 19-21; 16-17; 20; 21 - :images: | | | | - :height: 470px - ----- - -********************* -Maintain full control -********************* -Although we've abstracted the infrastructure, you still have full control when you need it: - -.. lit_tabs:: - :titles: Scheduler; Crontab syntax; Auto-scaling; Organized Python; Full terraform control; - :descriptions: Although you can use Python timers, we have a scheduler short-hand; You can also use full cron syntax; Code your own auto-scaling syntax (Lightning plays well with Kubernetes); *Remember* components organize ANY Python code which can even call external non-python scripts such as optimized C++ model servers ;Experts have the option to use terraform to configure Lightning clusters - :code_files: ./level_2_scripts/hello_app_scheduler.py; ./level_2_scripts/hello_app_cron.py; ./level_2_scripts/hello_app_auto_scale.py; ./level_2_scripts/organized_app_python.py; - :tab_rows: 4 - :highlights: 24; 24; 21, 24, 27, 28; 9, 16, 17 - :height: 700px - ----- - -************************* -Next: Review how to debug -************************* -The next levels does a 2 minute review to make sure you know how to debug a Lightning app. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 5: Debug a Lightning App - :description: Learn to debug a lightning app. - :button_link: debug_a_lightning_app.html - :col_css: col-md-12 - :height: 170 - :tag: 10 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/debug_a_lightning_app.rst b/docs/source-app/levels/intermediate/debug_a_lightning_app.rst deleted file mode 100644 index 8fdc7c67ae973..0000000000000 --- a/docs/source-app/levels/intermediate/debug_a_lightning_app.rst +++ /dev/null @@ -1,48 +0,0 @@ -############################## -Level 5: Debug A Lightning app -############################## -**Audience:** Users who want to debug a distributed app locally. - -**Prereqs:** You must have finished the :doc:`Basic levels <../basic/index>`. - ----- - -****************** -Enable breakpoints -****************** -To enable a breakpoint, use :func:`~lightning.app.pdb.set_trace()` (note direct python pdb support is work in progress and open to contributions). - -.. lit_tabs:: - :descriptions: Toy app; Add a breakpoint. When the program runs, it will stop at this line. - :code_files: ./debug_app_scripts/toy_app_1_component.py; ./debug_app_scripts/toy_app_1_component_pdb.py - :highlights: ; 7 - :enable_run: true - :tab_rows: 3 - :height: 350px - ----- - -********************************* -Next: Run a component in parallel -********************************* -Learn to run components in parallel to enable more powerful workflows. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 6: Run a Lightning component in parallel - :description: Learn when and how to run Components in parallel (asynchronous). - :button_link: run_lightning_work_in_parallel.html - :col_css: col-md-12 - :height: 150 - :tag: 15 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/debug_app_scripts/debug_app.py b/docs/source-app/levels/intermediate/debug_app_scripts/debug_app.py deleted file mode 100644 index cc2bbe16be35e..0000000000000 --- a/docs/source-app/levels/intermediate/debug_app_scripts/debug_app.py +++ /dev/null @@ -1,25 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.runners import MultiProcessRuntime - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent() - self.analyze = AnalyzeComponent() - - def run(self): - self.train.run("GPU machine 1") - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) -MultiProcessRuntime(app).dispatch() diff --git a/docs/source-app/levels/intermediate/debug_app_scripts/toy_app.py b/docs/source-app/levels/intermediate/debug_app_scripts/toy_app.py deleted file mode 100644 index 5f32ac07e2431..0000000000000 --- a/docs/source-app/levels/intermediate/debug_app_scripts/toy_app.py +++ /dev/null @@ -1,24 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp - - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent() - self.analyze = AnalyzeComponent() - - def run(self): - self.train.run("CPU machine 1") - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/debug_app_scripts/toy_app_1_component.py b/docs/source-app/levels/intermediate/debug_app_scripts/toy_app_1_component.py deleted file mode 100644 index e09df3ecb0d9c..0000000000000 --- a/docs/source-app/levels/intermediate/debug_app_scripts/toy_app_1_component.py +++ /dev/null @@ -1,17 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp - -class Component(LightningWork): - def run(self, x): - print(x) - - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component = Component() - - def run(self): - self.component.run('i love Lightning') - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/debug_app_scripts/toy_app_1_component_pdb.py b/docs/source-app/levels/intermediate/debug_app_scripts/toy_app_1_component_pdb.py deleted file mode 100644 index 6348f9c4ed46e..0000000000000 --- a/docs/source-app/levels/intermediate/debug_app_scripts/toy_app_1_component_pdb.py +++ /dev/null @@ -1,18 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.pdb import set_trace - -class Component(LightningWork): - def run(self, x): - print(x) - set_trace() - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component = Component() - - def run(self): - self.component.run('i love Lightning') - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/embed_web_ui_into_lightningwork.rst b/docs/source-app/levels/intermediate/embed_web_ui_into_lightningwork.rst deleted file mode 100644 index 354a82023f3a3..0000000000000 --- a/docs/source-app/levels/intermediate/embed_web_ui_into_lightningwork.rst +++ /dev/null @@ -1,40 +0,0 @@ -###################################### -Level 9: Embed graphical UIs into work -###################################### -**Audience:** Users who need to embed a Graphical UI in their Lightning Apps. - -**Prereqs:** You have finished :doc:`Level 8 `. - ----- - -.. include:: ../../workflows/add_web_ui/index_content.rst - ----- - -******************************************* -Next steps: Practice adapting app templates -******************************************* -One of the most exciting powers of Lightning is the ability -to start an app from a template, replace or add components -and build a powerful system. - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 10: Practice adapting app templates - :description: Practice starting apps from templates and evolving them by replacing or adding components. - :button_link: start_from_lightning_app_templates.html - :col_css: col-md-12 - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/index.rst b/docs/source-app/levels/intermediate/index.rst deleted file mode 100644 index ed5c3912acb0c..0000000000000 --- a/docs/source-app/levels/intermediate/index.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _intermediate_level: - -.. toctree:: - :maxdepth: 1 - :hidden: - - connect_lightning_components - debug_a_lightning_app - run_lightning_work_in_parallel - share_variables_between_lightning_components - share_files_between_components - embed_web_ui_into_lightningwork - start_from_lightning_app_templates - -################### -Intermediate skills -################### -Learn to coordinate 2+ components into workflows which we call Lightning apps. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 4: Coordinate 2+ components in a workflow - :description: Learn to coordinate 2_ components in a workflow which we call a Lightning app. - :button_link: connect_lightning_components.html - :col_css: col-md-6 - :height: 170 - :tag: 15 minutes - -.. displayitem:: - :header: Level 5: Debug a Lightning App - :description: Learn to debug a lightning app. - :button_link: debug_a_lightning_app.html - :col_css: col-md-6 - :height: 170 - :tag: 2 minutes - -.. displayitem:: - :header: Level 6: Run a Lightning component in parallel - :description: Learn when and how to run Components in parallel (asynchronous). - :button_link: run_lightning_work_in_parallel.html - :col_css: col-md-6 - :height: 150 - :tag: 10 minutes - -.. displayitem:: - :header: Level 7: Share variables between components - :description: Share variables between Lightning components. - :button_link: share_variables_between_lightning_components.html - :col_css: col-md-6 - :height: 150 - :tag: 15 minutes - -.. displayitem:: - :header: Level 8: Share files between components - :description: Learn how Drives share files between components - :button_link: share_files_between_components.html - :col_css: col-md-6 - :height: 150 - :tag: 20 minutes - -.. displayitem:: - :header: Level 9: Render a web UI with other components - :description: Learn how to embed graphical UIs like react, vue, streamlit and notebook UIs into a lightning workflow. - :button_link: embed_web_ui_into_lightningwork.html - :col_css: col-md-6 - :height: 150 - :tag: 15 minutes - -.. displayitem:: - :header: Level 10: Practice adapting app templates - :description: Practice starting apps from templates and evolving them by replacing or adding components. - :button_link: start_from_lightning_app_templates.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/level_12.rst b/docs/source-app/levels/intermediate/level_12.rst deleted file mode 100644 index 60c23909040a6..0000000000000 --- a/docs/source-app/levels/intermediate/level_12.rst +++ /dev/null @@ -1,12 +0,0 @@ -:orphan: - -###################### -Level 12: Flow vs Work -###################### -**Audience:** Users who need to do non trivial workloads in their apps. - -**Prereqs:** Level 8+ - ----- - -.. include:: ../../workflows/build_lightning_component/from_scratch_component_content.rst diff --git a/docs/source-app/levels/intermediate/level_2_scripts/code_run_cloud.bash b/docs/source-app/levels/intermediate/level_2_scripts/code_run_cloud.bash deleted file mode 100644 index 6594fe0ecac33..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/code_run_cloud.bash +++ /dev/null @@ -1 +0,0 @@ -lightning run app app.py --cloud diff --git a/docs/source-app/levels/intermediate/level_2_scripts/code_run_local.bash b/docs/source-app/levels/intermediate/level_2_scripts/code_run_local.bash deleted file mode 100644 index 8a00b45e132ca..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/code_run_local.bash +++ /dev/null @@ -1 +0,0 @@ -lightning run app app.py diff --git a/docs/source-app/levels/intermediate/level_2_scripts/hello_app.py b/docs/source-app/levels/intermediate/level_2_scripts/hello_app.py deleted file mode 100644 index 22edaa42add12..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/hello_app.py +++ /dev/null @@ -1,23 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('gpu')) - - def run(self): - self.train.run("CPU machine 1") - self.analyze.run("GPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/level_2_scripts/hello_app_auto_scale.py b/docs/source-app/levels/intermediate/level_2_scripts/hello_app_auto_scale.py deleted file mode 100644 index 5ea0903caa3c8..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/hello_app_auto_scale.py +++ /dev/null @@ -1,30 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('gpu')) - - def run(self): - # run() starts the machine - self.train.run("GPU machine 1") - - # stop() stops the machine - self.train.stop() - - # run analysis ONLY when machine 1 stopped - if self.train.status.STOPPED: - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/level_2_scripts/hello_app_cron.py b/docs/source-app/levels/intermediate/level_2_scripts/hello_app_cron.py deleted file mode 100644 index a5c1cdb8a7bee..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/hello_app_cron.py +++ /dev/null @@ -1,27 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('gpu')) - - def run(self): - # run training once - self.train.run("GPU machine 1") - - # run analysis once, then every hour again... - if self.schedule("5 4 * * *"): - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/level_2_scripts/hello_app_scheduler.py b/docs/source-app/levels/intermediate/level_2_scripts/hello_app_scheduler.py deleted file mode 100644 index a7ca0f415dfcf..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/hello_app_scheduler.py +++ /dev/null @@ -1,27 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('gpu')) - - def run(self): - # run training once - self.train.run("GPU machine 1") - - # run analysis once, then every hour again... - if self.schedule("hourly"): - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/level_2_scripts/organized_app_python.py b/docs/source-app/levels/intermediate/level_2_scripts/organized_app_python.py deleted file mode 100644 index e5e21d856f74d..0000000000000 --- a/docs/source-app/levels/intermediate/level_2_scripts/organized_app_python.py +++ /dev/null @@ -1,36 +0,0 @@ -# app.py -import subprocess -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class ExternalModelServer(LightningWork): - def run(self, x): - # compile - process = subprocess.Popen('g++ model_server.cpp -o model_server') - process.wait() - process = subprocess.Popen('./model_server') - process.wait() - -class LocustLoadTester(LightningWork): - def run(self, x): - cmd = f'locust --master-host {self.host} --master-port {self.port}' - process = subprocess.Popen(cmd) - process.wait() - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.serve = ExternalModelServer( - cloud_compute=CloudCompute('cpu'), parallel=True - ) - self.load_test = LocustLoadTester(cloud_compute=CloudCompute('cpu')) - - def run(self): - # start the server (on a CPU machine 1) - self.serve.run() - - # load testing when the server is up (on a separate cpu machine 2) - if self.serve.state.RUNNING: - self.load_test.run() - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/level_9.rst b/docs/source-app/levels/intermediate/level_9.rst deleted file mode 100644 index 344c321bac79b..0000000000000 --- a/docs/source-app/levels/intermediate/level_9.rst +++ /dev/null @@ -1,16 +0,0 @@ -:orphan: - -################### -Level 9: Event loop -################### -**Audience:** Users who want to build reactive Lightning Apps and move beyond DAGs. - -**Prereqs:** Level 8+ - ----- - -Drawing inspiration from modern web frameworks like `React.js `_, the Lightning App runs all flows in an **event loop** (forever), which is triggered several times a second after collecting any works' state change. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/lightning_loop.gif - -When running a Lightning App in the cloud, the ``LightningWork`` run on different machines. LightningWork communicates any state changes to the **event loop** which re-executes the flow with the newly-collected works' state. diff --git a/docs/source-app/levels/intermediate/run_lightning_work_in_parallel.rst b/docs/source-app/levels/intermediate/run_lightning_work_in_parallel.rst deleted file mode 100644 index 7a561893b2a35..0000000000000 --- a/docs/source-app/levels/intermediate/run_lightning_work_in_parallel.rst +++ /dev/null @@ -1,38 +0,0 @@ -############################################## -Level 6: Run a Lightning component in parallel -############################################## -**Audience:** Users who want to run a Lightning Component in parallel (asynchronously). - -**Prereqs:** You must have finished :doc:`Level 5 `. - ----- - -.. include:: ../../workflows/run_work_in_parallel_content.rst - ----- - -********************************************** -Next steps: Share variables between components -********************************************** -Now that you know how to run components in parallel, we'll learn to share variables -across components to simplify complex workflows. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 7: Share variables between components - :description: Learn to connect components - :col_css: col-md-12 - :button_link: share_variables_between_lightning_components.html - :height: 150 - :tag: 10 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/scripts/.storage/a b/docs/source-app/levels/intermediate/scripts/.storage/a deleted file mode 100644 index 1c6c4ccab0cee..0000000000000 Binary files a/docs/source-app/levels/intermediate/scripts/.storage/a and /dev/null differ diff --git a/docs/source-app/levels/intermediate/scripts/.storage/embeddings b/docs/source-app/levels/intermediate/scripts/.storage/embeddings deleted file mode 100644 index af3ee639fa570..0000000000000 Binary files a/docs/source-app/levels/intermediate/scripts/.storage/embeddings and /dev/null differ diff --git a/docs/source-app/levels/intermediate/scripts/a b/docs/source-app/levels/intermediate/scripts/a deleted file mode 100644 index 1c6c4ccab0cee..0000000000000 Binary files a/docs/source-app/levels/intermediate/scripts/a and /dev/null differ diff --git a/docs/source-app/levels/intermediate/scripts/comms_1.py b/docs/source-app/levels/intermediate/scripts/comms_1.py deleted file mode 100644 index d62d047c545f6..0000000000000 --- a/docs/source-app/levels/intermediate/scripts/comms_1.py +++ /dev/null @@ -1,18 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp - -class Component(LightningWork): - def run(self, x): - print(f'MACHINE 1: this string came from machine 0: "{x}"') - print('MACHINE 1: this string is on machine 1') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component = Component() - - def run(self): - x = 'hello from machine 0' - self.component.run(x) - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/scripts/debug_app.py b/docs/source-app/levels/intermediate/scripts/debug_app.py deleted file mode 100644 index cc2bbe16be35e..0000000000000 --- a/docs/source-app/levels/intermediate/scripts/debug_app.py +++ /dev/null @@ -1,25 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.runners import MultiProcessRuntime - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent() - self.analyze = AnalyzeComponent() - - def run(self): - self.train.run("GPU machine 1") - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) -MultiProcessRuntime(app).dispatch() diff --git a/docs/source-app/levels/intermediate/scripts/embeddings b/docs/source-app/levels/intermediate/scripts/embeddings deleted file mode 100644 index af3ee639fa570..0000000000000 Binary files a/docs/source-app/levels/intermediate/scripts/embeddings and /dev/null differ diff --git a/docs/source-app/levels/intermediate/scripts/toy_app.py b/docs/source-app/levels/intermediate/scripts/toy_app.py deleted file mode 100644 index 5f32ac07e2431..0000000000000 --- a/docs/source-app/levels/intermediate/scripts/toy_app.py +++ /dev/null @@ -1,24 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp - - - -class TrainComponent(LightningWork): - def run(self, x): - print(f'train a model on {x}') - -class AnalyzeComponent(LightningWork): - def run(self, x): - print(f'analyze model on {x}') - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent() - self.analyze = AnalyzeComponent() - - def run(self): - self.train.run("CPU machine 1") - self.analyze.run("CPU machine 2") - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/scripts/toy_payload.py b/docs/source-app/levels/intermediate/scripts/toy_payload.py deleted file mode 100644 index 473b450a5f56f..0000000000000 --- a/docs/source-app/levels/intermediate/scripts/toy_payload.py +++ /dev/null @@ -1,31 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp - - -class EmbeddingProcessor(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.embeddings = None - - def run(self): - print('PROCESSOR: Generating embeddings...') - fake_embeddings = [[1, 2, 3], [2, 3, 4]] - self.embeddings = storage.Payload(fake_embeddings) - -class EmbeddingServer(LightningWork): - def run(self, payload): - print('SERVER: Using embeddings from processor', payload) - embeddings = payload.value - print('serving embeddings sent from EmbeddingProcessor: ', embeddings) - -class WorkflowOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.processor = EmbeddingProcessor() - self.server = EmbeddingServer() - - def run(self): - self.processor.run() - self.server.run(self.processor.embeddings) - -app = LightningApp(WorkflowOrchestrator()) diff --git a/docs/source-app/levels/intermediate/scripts/two_comms_non_ml.py b/docs/source-app/levels/intermediate/scripts/two_comms_non_ml.py deleted file mode 100644 index 1a9e32589c110..0000000000000 --- a/docs/source-app/levels/intermediate/scripts/two_comms_non_ml.py +++ /dev/null @@ -1,36 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp -import time - -class A(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.msg_changed = False - self.new_msg = '' - - def run(self): - # pretend to train and save a checkpoint every 10 steps - for step in (range(1000)): - time.sleep(1.0) - if step % 10 == 0: - self.msg_changed = True - self.new_msg = f'A is at step: {step}' - print(self.new_msg) - -class B(LightningWork): - def run(self, msg): - print(f'B: message from A: {msg}') - -class Example(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.a = A(parallel=True) - self.b = B(parallel=True) - - def run(self): - self.a.run() - if self.a.msg_changed: - self.a.msg_changed = False - self.b.run(self.a.new_msg) - -app = LightningApp(Example()) diff --git a/docs/source-app/levels/intermediate/scripts/two_work_comms.py b/docs/source-app/levels/intermediate/scripts/two_work_comms.py deleted file mode 100644 index df37a468253fe..0000000000000 --- a/docs/source-app/levels/intermediate/scripts/two_work_comms.py +++ /dev/null @@ -1,35 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp -import time - -class TrainComponent(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.last_checkpoint_path = None - - def run(self): - # pretend to train and save a checkpoint every 10 steps - for step in (range(1000)): - time.sleep(1.0) - fake_loss = round(1/(step + 0.00001), 4) - print(f'{step=}: {fake_loss=} ') - if step % 10 == 0: - self.last_checkpoint_path = f'/some/path/{step=}_{fake_loss=}' - print(f'TRAIN COMPONENT: saved new checkpoint: {self.last_checkpoint_path}') - -class ModelDeploymentComponent(LightningWork): - def run(self, new_checkpoint): - print(f'DEPLOY COMPONENT: load new model from checkpoint: {new_checkpoint}') - -class ContinuousDeployment(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(parallel=True) - self.model_deployment = ModelDeploymentComponent(parallel=True) - - def run(self): - self.train.run() - if self.train.last_checkpoint_path: - self.model_deployment.run(self.train.last_checkpoint_path) - -app = LightningApp(ContinuousDeployment()) diff --git a/docs/source-app/levels/intermediate/share_files_between_components.rst b/docs/source-app/levels/intermediate/share_files_between_components.rst deleted file mode 100644 index 89c170f350a37..0000000000000 --- a/docs/source-app/levels/intermediate/share_files_between_components.rst +++ /dev/null @@ -1,34 +0,0 @@ -####################################### -Level 8: Share files between components -####################################### -**Audience:** Users who are moving large files such as artifacts or datasets. - -**Prereqs:** Level 6+ - ----- - -************************************************* -Next steps: Render a web UI with other components -************************************************* -Now that we know the key ways of sharing files and variables, -we'll apply it to embed web UIs alongside components. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 9: Render a web UI with other components - :description: Learn how to embed graphical UIs like react, vue, streamlit and notebook UIs into a lightning workflow. - :button_link: embed_web_ui_into_lightningwork.html - :col_css: col-md-12 - :height: 150 - :tag: 15 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/share_variables_between_lightning_components.rst b/docs/source-app/levels/intermediate/share_variables_between_lightning_components.rst deleted file mode 100644 index 81bbfbf718638..0000000000000 --- a/docs/source-app/levels/intermediate/share_variables_between_lightning_components.rst +++ /dev/null @@ -1,162 +0,0 @@ -########################################### -Level 7: Share variables between components -########################################### -**Audience:** Users who want to share variables and files across Lightning components. - -**Prereqs:** You must have finished `intermediate level 5+ `_. - ----- - -**************************************** -Send a variable from Flow to a Component -**************************************** -When a variable is defined on the LightningFlow (orchestrator), and -then it's passed into functions for the work components, under the hood -Lightning sends the variables across the machines for you automatically. - -.. lit_tabs:: - :descriptions: Remember this component may live on its own machine; The flow may be on a separate machine as well; This variable is on the flow machine; When passed to the work component, it is actually sent across the network under the hood.; When it prints here, it prints on the work component machine (not the flow machine); The second string was directly created on machine 1 - :code_files: ./scripts/comms_1.py; ./scripts/comms_1.py; ./scripts/comms_1.py; ./scripts/comms_1.py; ./scripts/comms_1.py; ./scripts/comms_1.py - :highlights: 4-7; 9-16; 15; 16; 6; 7; - :enable_run: true - :tab_rows: 3 - :height: 380px - -| - -.. collapse:: CLI output - - .. code-block:: - - $ lightning run app app.py --open-ui=false - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - MACHINE 1: this string came from machine 0: "hello from machine 0" - MACHINE 1: this string is on machine 1 - -| - -In this example, we learned that we can send variables to components like in regular Python. -On a local machine, it will behave like Python. When the workflow is distributed on the cloud, -it makes network calls under the hood, but still functions like Python to you. - ----- - -************************************** -Send a variable between two components -************************************** -A majority of workflows (especially in ML), require components to respond to a change in a component -likely running on a separate machine or even cluster. - -Example Continuous deployment: Every time a model saves a checkpoint, we redeploy a model: - -.. lit_tabs:: - :descriptions: Define a component that simulates training; Define a component that simulates deployment; Training will happen in parallel over a long period; The deployment server also runs in parallel forever; Start training in parallel (could take months); Whenever the model has a checkpoint deploy; When the checkpoint is updated, model re-deploys - :code_files: ./scripts/two_work_comms.py; ./scripts/two_work_comms.py; ./scripts/two_work_comms.py; ./scripts/two_work_comms.py; ./scripts/two_work_comms.py; ./scripts/two_work_comms.py; ./scripts/two_work_comms.py - :highlights: 5-18; 20-22; 27; 28; 31; 32, 33; 33 - :enable_run: true - :tab_rows: 3 - :height: 690px - -| - -.. collapse:: CLI output: - - .. code:: - - $ lightning run app app.py --open-ui=false - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - step=0: fake_loss=100000.0 - TRAIN COMPONENT: saved new checkpoint: /some/path/step=0_fake_loss=100000.0 - step=1: fake_loss=1.0 - DEPLOY COMPONENT: load new model from checkpoint: /some/path/step=0_fake_loss=100000.0 - step=2: fake_loss=0.5 - step=3: fake_loss=0.3333 - step=4: fake_loss=0.25 - step=5: fake_loss=0.2 - step=6: fake_loss=0.1667 - step=7: fake_loss=0.1429 - step=8: fake_loss=0.125 - step=9: fake_loss=0.1111 - step=10: fake_loss=0.1 - TRAIN COMPONENT: saved new checkpoint: /some/path/step=10_fake_loss=0.1 - DEPLOY COMPONENT: load new model from checkpoint: /some/path/step=10_fake_loss=0.1 - step=11: fake_loss=0.0909 - step=12: fake_loss=0.0833 - step=13: fake_loss=0.0769 - step=14: fake_loss=0.0714 - step=15: fake_loss=0.0667 - step=16: fake_loss=0.0625 - step=17: fake_loss=0.0588 - step=18: fake_loss=0.0556 - step=19: fake_loss=0.0526 - step=20: fake_loss=0.05 - TRAIN COMPONENT: saved new checkpoint: /some/path/step=20_fake_loss=0.05 - DEPLOY COMPONENT: load new model from checkpoint: /some/path/step=20_fake_loss=0.05 - step=21: fake_loss=0.0476 - ----- - -******************************************** -Send a large variable between two components -******************************************** -For large variables such as arrays, tensors, embeddings and so on, use Payload to enable -transferring them across components. - -.. lit_tabs:: - :descriptions: Let's define a component to simulate generating embeddings (from a DB, feature store, etc...); This component simulates a server that will use the embeddings.; Run the component to generate the embeddings; Simulate embeddings as an array. Here you would query a DB, load from a feature store or disk or even use a neural network to extract the embedding.; Allow the embeddings to be transferred efficiently by wrapping them in the Payload object.; Pass the variable to the EmbeddingServer (just the pointer).; The data gets transferred once you use the .value attribute in the other component. - :code_files: ./scripts/toy_payload.py; ./scripts/toy_payload.py; ./scripts/toy_payload.py; ./scripts/toy_payload.py; ./scripts/toy_payload.py; ./scripts/toy_payload.py; ./scripts/toy_payload.py; - :highlights: 5-13; 15-19; 28; 12; 13; 29; 18 - :enable_run: true - :tab_rows: 3 - :height: 600px - -| - -.. collapse:: CLI output - - .. code:: - - $ lightning run app app.py --open-ui=false - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - PROCESSOR: Generating embeddings... - SERVER: Using embeddings from processor - serving embeddings sent from EmbeddingProcessor: [[1, 2, 3], [2, 3, 4]] - -| - -The payload object keeps the data on the machine and passes a pointer -to the data around the app until the data is needed by a component. - ----- - -****************************************** -Next steps: Share files between components -****************************************** -Now that you know how to run components in parallel, we'll learn to share variables -across components to simplify complex workflows. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 8: Share files between components - :description: Learn to share files between components. - :col_css: col-md-12 - :button_link: share_files_between_components.html - :height: 150 - :tag: 10 minutes - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/start_from_lightning_app_templates.rst b/docs/source-app/levels/intermediate/start_from_lightning_app_templates.rst deleted file mode 100644 index 2b5ae06e48022..0000000000000 --- a/docs/source-app/levels/intermediate/start_from_lightning_app_templates.rst +++ /dev/null @@ -1,34 +0,0 @@ -############################################ -Level 10: Start from lightning app templates -############################################ -**Audience:** All users who want to move quickly with Lightning - -**Prereqs:** You have finished :doc:`Level 9 `. - ----- - -**************************************************** -Next step: Learn to build powerful nested components -**************************************************** -Now that you can build powerful apps, learn to build nested components -that can do things like start dynamic works and connect to each other -via networking or CLI commands. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Advanced skills - :description: Learn to build nested components with advanced functionality. - :button_link: ../advanced/index.html - :col_css: col-md-12 - :height: 170 - -.. raw:: html - -
-
diff --git a/docs/source-app/make.bat b/docs/source-app/make.bat deleted file mode 100644 index 9b565142aecbf..0000000000000 --- a/docs/source-app/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=../build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/source-app/moving_to_the_cloud.rst b/docs/source-app/moving_to_the_cloud.rst deleted file mode 100644 index a4d57e9fc026c..0000000000000 --- a/docs/source-app/moving_to_the_cloud.rst +++ /dev/null @@ -1,122 +0,0 @@ -:orphan: - -.. _moving_to_the_cloud: - -#################### -Moving to the Cloud -#################### - -.. warning:: This is in progress and not yet fully supported. - -In the :ref:`quick_start` guide, you learned how to implement a simple app -that trains an image classifier and serve it once trained. - -In this tutorial, you'll learn how to extend that application so that it works seamlessly -both locally and in the cloud. - ----- - -******************************** -Step 1: Distributed Application -******************************** - - -Distributed Storage -^^^^^^^^^^^^^^^^^^^ - -When running your application in a fully-distributed setting, the data available on one machine won't necessarily be available on another. - -To solve this problem, Lightning introduces the :class:`~lightning.app.storage.path.Path` object. -This ensures that your code can run both locally and in the cloud. - -The :class:`~lightning.app.storage.path.Path` object keeps track of the work which creates -the path. This enables Lightning to transfer the files correctly in a distributed setting. - -Instead of passing a string representing a file or directory, Lightning simply wraps -them into a :class:`~lightning.app.storage.path.Path` object and makes them an attribute of your LightningWork. - -Without doing this conscientiously for every single path, your application will fail in the cloud. - -In the example below, a file written by **SourceFileWork** is being transferred by the flow -to the **DestinationFileAndServeWork** work. The Path object is the reference to the file. - -.. literalinclude:: ../../examples/app/boring/app.py - :emphasize-lines: 5, 22, 28, 48 - -In the ``scripts/serve.py`` file, we are creating a **FastApi Service** running on port ``1111`` -that returns the content of the file received from **SourceFileWork** when -a post request is sent to ``/file``. - -.. literalinclude:: ../../examples/app/boring/scripts/serve.py - :emphasize-lines: 21, 23-26 - ----- - -Distributed Frontend -^^^^^^^^^^^^^^^^^^^^ - -In the above example, the **FastAPI Service** was running on one machine, -and the frontend UI in another. - -In order to assemble them, you need to do two things: - -* Provide **port** argument to your work's ``__init__`` method to expose a single service. - -Here's how to expose the port: - -.. literalinclude:: ../../examples/app/boring/app.py - :emphasize-lines: 8 - :lines: 33-44 - - -And here's how to expose your services within the ``configure_layout`` flow hook: - -.. literalinclude:: ../../examples/app/boring/app.py - :emphasize-lines: 5 - :lines: 53-57 - -In this example, we're appending ``/file`` to our **FastApi Service** url. -This means that our ``Boring Tab`` triggers the ``get_file_content`` from the **FastAPI Service** -and embeds its content as an `IFrame `_. - -.. literalinclude:: ../../examples/app/boring/scripts/serve.py - :lines: 23-26 - - -Here's a visualization of the application described above: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/storage_ui.gif - :alt: Storage API Animation - :width: 100 % - ----- - -***************************** -Step 2: Scalable Application -***************************** - -The benefit of defining long-running code inside a -:class:`~lightning.app.core.work.LightningWork` -component is that you can run it on different hardware -by providing :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute` to -the ``__init__`` method of your :class:`~lightning.app.core.work.LightningWork`. - -By adapting the :ref:`quick_start` example as follows, you can easily run your component on multiple GPUs: - - -Without doing much, you’re now running a script on its own cluster of machines! 🤯 - ----- - -***************************** -Step 3: Resilient Application -***************************** - -We designed Lightning with a strong emphasis on supporting failure cases. -The framework shines when the developer embraces our fault-tolerance best practices, -enabling them to create ML applications with a high degree of complexity as well as a strong support -for unhappy cases. - -An entire section would be dedicated to this concept. - -TODO diff --git a/docs/source-app/quickstart.rst b/docs/source-app/quickstart.rst deleted file mode 100644 index 99872c5f8ae46..0000000000000 --- a/docs/source-app/quickstart.rst +++ /dev/null @@ -1,125 +0,0 @@ -:orphan: - -.. _quick_start: - -############ -Quick Start -############ - -In this guide, we'll run an application that trains -an image classification model with the `MNIST Dataset `_, -and uses `Gradio `_ to serve it. - ----- - -********************** -Step 1 - Installation -********************** - -First, you'll need to install Lightning. You can find the complete guide here. - -Then, you'll need to install the `Lightning Quick Start package `_. - -.. code-block:: bash - - lightning install app lightning/quick-start - -And download the training script used by the App: - - ----- - -********************** -Step 2 - Run the app -********************** - -To run your app, copy the following command to your local terminal: - -.. code-block:: bash - - lightning run app app.py - -And that's it! - -.. admonition:: You should see the app logs in your terminal. - :class: dropdown - - .. code-block:: console - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - - Global seed set to 42 - - GPU available: True (mps), used: False - TPU available: False, using: 0 TPU cores - - | Name | Type | Params | In sizes | Out sizes - ------------------------------------------------------------------ - 0 | model | Net | 1.2 M | [1, 1, 28, 28] | [1, 10] - 1 | val_acc | Accuracy | 0 | ? | ? - ------------------------------------------------------------------ - 1.2 M Trainable params - 0 Non-trainable params - 1.2 M Total params - Epoch 4: 100%|█████████████████████████| 16/16 [00:00<00:00, 32.31it/s, loss=0.0826, v_num=0] - `Trainer.fit` stopped: `max_epochs=5` reached. - - Running on local URL: http://127.0.0.1:62782/ - ... - - -The app will open your browser and show an interactive demo: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/qiuck-start-tensorboard-tab.png - :alt: Quick Start UI - Model Training Tab - :width: 100 % - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/quick-start-gradio-tab.png - :alt: Quick Start UI - Interactive Demo Tab - :width: 100 % - ----- - -This app behind the scenes -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This application has one flow component which coordinates two works executing their own python script. -Once the training is finished, the trained model weights are passed to the serve component. - - -Here is how the components of a Lightning app are structured: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/quick_start_components.gif - :alt: Quick Start Application - :width: 100 % - -Here is the application timeline: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/timeline.gif - :alt: Quick Start Timeline Application - :width: 100 % - ----- - -************************************** -Steps 3 - Build your app in the cloud -************************************** - -Simply add ``--cloud`` to run this application in the cloud 🤯 - -.. code-block:: bash - - lightning run app app.py --cloud - -Congratulations! You've now run your first application with Lightning. - ----- - -*********** -Next Steps -*********** - -To learn how to build and modify apps, go to the :ref:`basics`. - -To learn how to create UIs for your apps, read :ref:`ui_and_frontends`. diff --git a/docs/source-app/testing.rst b/docs/source-app/testing.rst deleted file mode 100644 index 752291541000e..0000000000000 --- a/docs/source-app/testing.rst +++ /dev/null @@ -1,155 +0,0 @@ -:orphan: - -.. _testing: - -####################### -Productionize your Apps -####################### - -.. TODO: Cleanup - -At the core of our system is an integration testing framework that will allow for a first-class experience creating integration tests for Lightning Apps. This document will explain how we can create a lightning app test, how we can execute it, and where to find more information. - ----- - -*********** -Philosophy -*********** - -Testing a Lightning app is unique. It is a superset of an application that converges machine learning, API development, and UI development. With that in mind, there are several philosophies (or "best practices") that you should adhere to: - - -#. **Control your app state** - With integration and end to end tests, you have the capabilities of controlling your app's state through dependency injection. Use it! -#. **Integration focuses on the work, End to End focuses on the app** - When writing tests, think of the depth and breath of what you are writing. Write many integration tests since they are relatively cheap, while keeping the end to end tests for holistic app testing. -#. **Don't overthink it** - What needs to be tested? What is the order of risk? These are the questions you should build with before writing your first line of code. Writing tests for the sake of writing tests is an exercise in futility. Write meaningful, impactful tests. -#. **Test Isolation** - Write your tests in an isolated manner. No two tests should ever depend on each other. -#. **Use your framework** - Testing apps should be framework agnostic. -#. **Have fun!** - At the heart of testing is experimentation. Like any experiment, tests begin with a hypothesis of workability, but you can extend that to be more inclusive. Ask the question, write the test to answer your question, and make sure you have fun while doing it. - ----- - -**************************************** -Anatomy of a Lightning integration test -**************************************** - -The following is a PyTest example of an integration test using the ``lightning.app.testing`` module. - -.. code-block:: python - - import os - - from lightning.app import _PROJECT_ROOT - from lightning.app.testing import application_testing, LightningTestApp - from lightning.app.utilities.enum import AppStage - - - class TestLightningAppInt(TestLightningApp): - def run_once(self) -> bool: - if self.root.counter > 1: - print("V0 App End") - self.stage = AppStage.STOPPING - return True - return super().run_once() - - - def test_v0_app_example(): - command_line = [ - os.path.join(_PROJECT_ROOT, "examples/app_v0/app.py"), - "--blocking", - "False", - "--multiprocess", - "--open-ui", - "False", - ] - result = application_testing(TestLightningAppInt, command_line) - assert "V0 App End" in str(result.stdout_bytes) - assert result.exit_code == 0 - ----- - -Setting up the app -^^^^^^^^^^^^^^^^^^ - -Lightning apps are unique in that they represent a full stack model for your machine learning application. To be clear, the integration tests are *NOT* going to touch the UI flow. Instead we inject your application with helper methods that, when executed, can assist in validating your application. - -To get started, you simply need to import the following: - -.. code-block:: python - - from lightning.app.testing import application_testing, LightningTestApp - -We will discuss ``application_testing`` in a bit, but first let's review the structure of ``LightningTestApp``. - ----- - -LightningTestApp -^^^^^^^^^^^^^^^^^ - -The :class:`lightning.app.testing.testing.LightningTestApp` class is available to use for provisioning and setting up your testing needs. Note that you do not need this class to move forward with testing. Any application that inherits ``LightningApp`` should suffice as long as you override the correct methods. Reviewing the TestLightnigApp we see some overrides that are already there. Please revuew the class for more information. - -.. code-block:: python - - class TestLightningAppInt(LightningTestApp): - def run_once(self) -> bool: - if self.root.counter > 1: - print("V0 App End") - self.stage = AppStage.STOPPING - return True - return super().run_once() - -We create a test class overriding the ``run_once`` function. This function helps control the flow of your application and is ran first. In this example we are calling ``self.root.counter`` and checking if the job has executed once. If so, we want to print out ``V0 App End`` and set the ``self.stage`` to ``AppStage.STOPPING``. This is how we control the flow through state. Your situation might be different, so experiment and see what you can do! - -Besides ``run_once`` there are a few other overrides available: - - -* ``on_before_run_once`` - This runs before your ``run_once`` function kicks off. You can set up your application pre-conditions here. -* ``on_after_run_once`` - Similar to ``on_before_run_once`` but after the ``run_once`` method is called. - -These methods will skew your tests, so use them when needed. - ----- - -The Test -^^^^^^^^ - -We provide ``application_testing`` as a helper function to get your application up and running for testing. It uses ``click``\ 's invocation tooling underneath. - -.. code-block:: - - command_line = [ - os.path.join(_PROJECT_ROOT, "examples/app_v0/app.py"), - "--blocking", - "False", - "--open-ui", - "False", - ] - -First in the list for ``command_line`` is the location of your script. It is an external file. In this example we have ``_PROJECT_ROOT`` but this is *not* a helper constant for you to utilize. You will need to provide the location yourself. - -Next there are a couple of options you can leverage: - -* ``blocking`` - Blocking is an app status that says "Do not run until I click run in the UI". For our integration test, since we are not using the UI, we are setting this to "False". -* ``open-ui`` - We set this to false since this is the routine that opens a browser for your local execution. - -Once you have your commandline ready, you will then be able to kick off the test and gather results: - -.. code-block:: python - - result = application_testing(TestLightningAppInt, command_line) - -As mentioned earlier, ``application_testing`` is a helper method that allows you to inject your TestLightningApp class (with overrides) and the commandline flags. Once the process is done it returns the results back for parsing. - -.. code-block:: python - - assert "V0 App End" in str(result.stdout_bytes) - assert result.exit_code == 0 - -Since we injected "V0 App End" to the end of our test flow. The state was changed to ``AppStatus.STOPPING`` which means the process is done. Finally, we check the result's exit code to make sure that we did not throw an error during execution. - ----- - -************ -End to End -************ - -TODO diff --git a/docs/source-app/ui_and_frontends.rst b/docs/source-app/ui_and_frontends.rst deleted file mode 100644 index 3ffa5f983e7d8..0000000000000 --- a/docs/source-app/ui_and_frontends.rst +++ /dev/null @@ -1,23 +0,0 @@ -:orphan: - -.. _ui_and_frontends: - -################ -UI and Frontends -################ - - -The Lightning framework allows you to create customized, interactive UIs with the framework of your choice. - -You can easily embed other tools and services (like a GitHub repo, a `FastAPI Service `_, an `arXiv `_ paper or a `Dask Cluster `_ Admin page), or create a complete UI from scratch. - - -To get started, you can use built-in templates for the following frameworks: - -* `React.js `_ -* `StreamLit `_ - - - -To keep learning about Lightning, check out :ref:`moving_to_the_cloud`. -This section covers best practices to seamlessly make your Lightning code work both locally and in the cloud. diff --git a/docs/source-app/workflows/access_app_state.rst b/docs/source-app/workflows/access_app_state.rst deleted file mode 100644 index 8f99534bd239b..0000000000000 --- a/docs/source-app/workflows/access_app_state.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _access_app_state: - -################ -Access App State -################ - -**Audience:** Users who want to know how the App State can be accessed. - -**Level:** Basic - -********************** -What is the App State? -********************** - -In Lightning, each component is stateful and their state is composed of all attributes defined within their **__init__** method. - -The **App State** is the collection of all the components' states forming the App. - -************************************ -What is special about the App State? -************************************ - -The **App State** is always up-to-date, even running an App in the cloud on multiple machines. -This means that every time an attribute is modified in a Work, that information is automatically -broadcasted to the Flow. With this mechanism, any Component can **react** to any other -Component's **state changes** through the Flow and complex systems can be easily implemented. -Lightning requires a state based driven mindset when implementing the Flow. - -*************************************** -When do I need to access the App State? -*************************************** - -As a user, you are interacting with your component attributes, so most likely, -you won't need to access the Component's state directly, but it can be helpful to -understand how the state works under the hood. - -For example, here we define a **Flow** component and **Work** component, where the Work increments a counter indefinitely and the Flow prints its state which contains the Work. - -You can easily check the state of your entire App as follows: - -.. literalinclude:: ../code_samples/quickstart/app_01.py - -Run the App with: - -.. code-block:: bash - - lightning run app docs/quickstart/app_01.py - -And here's the output you get when running the App using **Lightning CLI**: - -.. code-block:: console - - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - State: {'works': {'w': {'vars': {'counter': 1}}}} - State: {'works': {'w': {'vars': {'counter': 2}}}} - State: {'works': {'w': {'vars': {'counter': 3}}}} - State: {'works': {'w': {'vars': {'counter': 3}}}} - State: {'works': {'w': {'vars': {'counter': 4}}}} - ... diff --git a/docs/source-app/workflows/add_components.rst b/docs/source-app/workflows/add_components.rst deleted file mode 100644 index 3edff4c1c0c7d..0000000000000 --- a/docs/source-app/workflows/add_components.rst +++ /dev/null @@ -1,31 +0,0 @@ -:orphan: - -########################### -Add a component to your app -########################### -**Audience:** Users looking to expand the functionality of their Lightning apps. - ----- - -******************* -Install a component -******************* - -Any Lightning component can be installed with: - -.. code:: python - - lightning install component org/the-component-name - -`Browse all community-built components here `_. - -.. note:: Components are being populated daily - ----- - -********************** -Contribute a component -********************** -One of the first principles of the Lightning community is to code something *once* for the benefit or everyone! - -To contribute a component, :doc:`follow this guide `. diff --git a/docs/source-app/workflows/add_server/any_server.rst b/docs/source-app/workflows/add_server/any_server.rst deleted file mode 100644 index 948ae22a2abc0..0000000000000 --- a/docs/source-app/workflows/add_server/any_server.rst +++ /dev/null @@ -1,187 +0,0 @@ -######################### -Enable any server (basic) -######################### -**Audience:** Users who want to enable an arbitrary server/UI. - -**Prereqs:** Basic python knowledge. - ----- - -***************** -What is a server? -***************** -A server is a program that enables other programs or users to connect to it. As long as your server can listen on a port, -you can enable it with a Lightning App. - ----- - -*************************** -Add a server to a component -*************************** -Any server that listens on a port, can be enabled via a work. For example, here's a plain python server: - -.. code:: python - :emphasize-lines: 11-12 - - import socketserver - from http import HTTPStatus, server - - - class PlainServer(server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(HTTPStatus.OK) - self.end_headers() - # Data must be passed as bytes to the `self.wfile.write` call - html = b"

Hello lit world

" - self.wfile.write(html) - - - httpd = socketserver.TCPServer(("localhost", "3000"), PlainServer) - httpd.serve_forever() - -To enable the server inside the component, start the server in the run method and use the ``self.host`` and ``self.port`` properties: - -.. code:: python - :emphasize-lines: 14-15 - - import lightning as L - import socketserver - from http import HTTPStatus, server - - - class PlainServer(server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(HTTPStatus.OK) - self.end_headers() - # Data must be passed as bytes to the `self.wfile.write` call - html = b"

Hello lit world

" - self.wfile.write(html) - - - class LitServer(L.LightningWork): - def run(self): - httpd = socketserver.TCPServer((self.host, self.port), PlainServer) - httpd.serve_forever() - ----- - -************************************** -Route the server in the root component -************************************** -The final step, is to tell the Root component in which tab to render this component's output: -In this case, we render the ``LitServer`` output in the ``home`` tab of the application. - -.. code:: python - :emphasize-lines: 20, 23, 28 - - import lightning as L - import socketserver - from http import HTTPStatus, server - - - class PlainServer(server.SimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(HTTPStatus.OK) - self.end_headers() - # Data must be passed as bytes to the `self.wfile.write` call - html = b"

Hello lit world

" - self.wfile.write(html) - - - class LitServer(L.LightningWork): - def run(self): - httpd = socketserver.TCPServer((self.host, self.port), PlainServer) - httpd.serve_forever() - - - class Root(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_server = LitServer(parallel=True) - - def run(self): - self.lit_server.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_server} - return tab1 - - - app = L.LightningApp(Root()) - -We use the ``parallel=True`` argument of ``LightningWork`` to run the server in parallel -while the rest of the Lightning App runs everything else. - ----- - -*********** -Run the app -*********** -Start the app to see your new UI! - -.. code:: bash - - lightning_app run app app.py - -To run the app on the cloud, use the ``--cloud`` argument. - -.. code:: bash - - lightning_app run app app.py --cloud - ----- - -***************************************** -Interact with a component from the server -***************************************** - -.. TODO:: how do we do this? - - ----- - -***************************************** -Interact with the server from a component -***************************************** - -.. TODO:: how do we do this? - ----- - -******** -Examples -******** -Here are a few example apps that expose a server via a component: - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example: Tensorboard - :description: TODO - :col_css: col-md-4 - :button_link: example_app.html - :height: 150 - -.. displayitem:: - :header: Example: Streamlit - :description: TODO - :col_css: col-md-4 - :button_link: example_app.html - :height: 150 - -.. displayitem:: - :header: Example: React - :description: TODO - :col_css: col-md-4 - :button_link: example_app.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_server/flask_basic.rst b/docs/source-app/workflows/add_server/flask_basic.rst deleted file mode 100644 index 38ca282346248..0000000000000 --- a/docs/source-app/workflows/add_server/flask_basic.rst +++ /dev/null @@ -1,155 +0,0 @@ -############################### -Add a web UI with Flask (basic) -############################### -**Audience:** Users who want to enable a flask app within a component. - -**Prereqs:** Basic python knowledge. - ----- - -************** -What is Flask? -************** -Flask is a web framework, that lets you develop web applications in Python easily. - ----- - -************************ -Add Flask to a component -************************ -First, define your flask app as you normally would without Lightning: - -.. code:: python - :emphasize-lines: 9 - - from flask import Flask - - flask_app = Flask(__name__) - - - @flask_app.route("/") - def hello(): - return "Hello, World!" - - - flask_app.run(host="0.0.0.0", port=80) - -To enable the server inside the component, start the Flask server in the run method and use the ``self.host`` and ``self.port`` properties: - -.. code:: python - :emphasize-lines: 12 - - import lightning as L - from flask import Flask - - - class LitFlask(L.LightningWork): - def run(self): - flask_app = Flask(__name__) - - @flask_app.route("/") - def hello(): - return "Hello, World!" - - flask_app.run(host=self.host, port=self.port) - ----- - -************************************** -Route the server in the root component -************************************** -The final step, is to tell the Root component in which tab to render this component's output: -In this case, we render the ``LitFlask`` output in the ``home`` tab of the application. - -.. code:: python - :emphasize-lines: 17, 23 - - import lightning as L - from flask import Flask - - - class LitFlask(L.LightningWork): - def run(self): - flask_app = Flask(__name__) - - @flask_app.route("/") - def hello(): - return "Hello, World!" - - flask_app.run(host=self.host, port=self.port) - - - class Root(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_flask = LitFlask(parallel=True) - - def run(self): - self.lit_flask.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_flask} - return tab1 - - - app = L.LightningApp(Root()) - -We use the ``parallel=True`` argument of ``LightningWork`` to run the server in the background -while the rest of the Lightning App runs everything else. - ----- - -*********** -Run the app -*********** -Start the app to see your new UI! - -.. code:: bash - - lightning run app app.py - -To run the app on the cloud, use the ``--cloud`` argument. - -.. code:: bash - - lightning run app app.py --cloud - ----- - -******** -Examples -******** -Here are a few example apps that expose a Flask server via a component: - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: TODO - :col_css: col-md-4 - :button_link: example_app.html - :height: 150 - -.. displayitem:: - :header: Example 2 - :description: TODO - :col_css: col-md-4 - :button_link: example_app.html - :height: 150 - -.. displayitem:: - :header: Example 3 - :description: TODO - :col_css: col-md-4 - :button_link: example_app.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_server/index.rst b/docs/source-app/workflows/add_server/index.rst deleted file mode 100644 index 1429b08679b2a..0000000000000 --- a/docs/source-app/workflows/add_server/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -################################### -Run a server within a Lightning App -################################### -Any type of server can run inside a Lightning App. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/add_server/index_content.rst b/docs/source-app/workflows/add_server/index_content.rst deleted file mode 100644 index d3623698cfb19..0000000000000 --- a/docs/source-app/workflows/add_server/index_content.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :hidden: - - any_server - flask_basic - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Any server - :description: Learn how to enable any server inside a Lightning App. - :col_css: col-md-6 - :button_link: any_server.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Flask - :description: Learn how to add a Flask server inside a Lightning App. - :col_css: col-md-6 - :button_link: flask_basic.html - :height: 150 - :tag: basic - - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_link.rst b/docs/source-app/workflows/add_web_link.rst deleted file mode 100644 index 01ffdf62ef3e3..0000000000000 --- a/docs/source-app/workflows/add_web_link.rst +++ /dev/null @@ -1,54 +0,0 @@ -############## -Add a web link -############## -**Audience:** Users who want to link to other pages from their app. - ----- - -************** -Add a url link -************** -In this example we'll replicate |urls_link|. - -To add a url link to an app, simply specify it in the ``configure_layout`` method -and connect the UIs. Create a file named **app.py** with this code: - -.. |urls_link| raw:: html - - the app running here - -.. code:: python - :emphasize-lines: 7,11 - - import lightning as L - - class LitApp(L.LightningFlow): - def configure_layout(self): - tab_1 = { - "name": "Logger", - "content": "https://bit.ly/tb-aasae" - } - tab_2 = { - "name": "Paper", - "content": "https://arxiv.org/pdf/2107.12329.pdf" - } - return tab_1, tab_2 - - app = L.LightningApp(LitApp()) - ----- - -*********** -Run the app -*********** -Run the app locally to see it! - -.. code:: python - - lightning run app app.py - -Now run it on the cloud as well: - -.. code:: python - - lightning run app app.py --cloud diff --git a/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst b/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst deleted file mode 100644 index 095dee362bf48..0000000000000 --- a/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst +++ /dev/null @@ -1,6 +0,0 @@ -:orphan: - -########################################### -Add a web UI with Angular.js (intermediate) -########################################### -coming... diff --git a/docs/source-app/workflows/add_web_ui/dash/basic.rst b/docs/source-app/workflows/add_web_ui/dash/basic.rst deleted file mode 100644 index 4316fc13a1992..0000000000000 --- a/docs/source-app/workflows/add_web_ui/dash/basic.rst +++ /dev/null @@ -1,221 +0,0 @@ -############################## -Add a web UI with Dash (basic) -############################## -**Audience:** Users who want to add a web UI with Dash by Plotly. - -**Prereqs:** Basic python knowledge. - ----- - -************* -What is Dash? -************* -`Dash `_ is the original low-code framework for rapidly building data apps in Python, R, Julia, and F# (experimental). - -Install Dash with: - -.. code:: bash - - pip install dash - ----- - -************************ -Create the dash demo app -************************ - -To explain how to use Dash with Lightning, let's build a simple app with Dash. - - -.. - To explain how to use Dash with Lightning, let's replicate the |dash_link|. - - .. |dash_link| raw:: html - - example running here - -In the next few sections we'll build an app step-by-step. -First **create a file named app.py** with the app content: - -.. code:: bash - - import lightning as L - import dash - import plotly.express as px - - class LitDash(L.LightningWork): - def run(self): - dash_app = dash.Dash(__name__) - X = [1, 2, 3, 4, 5, 6] - Y = [2, 4, 8, 16, 32, 64] - fig = px.line(x=X, y=Y) - - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) - - dash_app.run_server(host=self.host, port=self.port) - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash(parallel=True) - - def run(self): - self.lit_dash.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - app = L.LightningApp(LitApp()) - - -add 'dash' to a requirements.txt file: - -.. code:: bash - - echo "dash" >> requirements.txt - -this is a best practice to make apps reproducible. - ----- - -*********** -Run the app -*********** -Run the app locally to see it! - -.. code:: python - - lightning run app app.py - -Now run it on the cloud as well: - -.. code:: python - - lightning run app app.py --cloud - ----- - -************************ -Step-by-step walkthrough -************************ -In this section, we explain each part of this code in detail. - ----- - -0. Define a Dash app -^^^^^^^^^^^^^^^^^^^^ -First, find the dash app you want to integrate. In this example, that app looks like: - -.. code:: python - - import dash - import plotly.express as px - - dash_app = dash.Dash(__name__) - X = [1, 2, 3, 4, 5, 6] - Y = [2, 4, 8, 16, 32, 64] - fig = px.line(x=X, y=Y) - - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) - - dash_app.run_server(host='0.0.0.0', port=80) - -This dash app plots a simple line curve along with some HTMlapp. -`Visit the Dash documentation for the full API `_. - ----- - -1. Add Dash to a component -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Add the dash app to the run method of a ``LightningWork`` component and run the server on that component's **host** and **port**: - -.. code:: python - :emphasize-lines: 6, 18 - - import lightning as L - import dash - import plotly.express as px - - class LitDash(L.LightningWork): - def run(self): - dash_app = dash.Dash(__name__) - X = [1, 2, 3, 4, 5, 6] - Y = [2, 4, 8, 16, 32, 64] - fig = px.line(x=X, y=Y) - - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) - - dash_app.run_server(host=self.host, port=self.port) - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash(parallel=True) - - def run(self): - self.lit_dash.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - app = L.LightningApp(LitApp()) - ----- - -2. Route the UI in the root component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The final step, is to tell the Root component in which tab to render this component's UI. -In this case, we render the ``LitDash`` UI in the ``home`` tab of the application. - -.. code:: python - :emphasize-lines: 23, 29 - - import lightning as L - import dash - import plotly.express as px - - class LitDash(L.LightningWork): - def run(self): - dash_app = dash.Dash(__name__) - X = [1, 2, 3, 4, 5, 6] - Y = [2, 4, 8, 16, 32, 64] - fig = px.line(x=X, y=Y) - - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) - - dash_app.run_server(host=self.host, port=self.port) - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash(parallel=True) - - def run(self): - self.lit_dash.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - app = L.LightningApp(LitApp()) - -We use the ``parallel=True`` argument of ``LightningWork`` to run the server in the background -while the rest of the Lightning App runs everything else. diff --git a/docs/source-app/workflows/add_web_ui/dash/index.rst b/docs/source-app/workflows/add_web_ui/dash/index.rst deleted file mode 100644 index 5abb444c8e9a9..0000000000000 --- a/docs/source-app/workflows/add_web_ui/dash/index.rst +++ /dev/null @@ -1,84 +0,0 @@ -:orphan: - -.. toctree:: - :maxdepth: 1 - :hidden: - - basic - intermediate - -###################### -Add a web UI with Dash -###################### - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1: Connect Dash - :description: Learn how to connect a Dash app. - :col_css: col-md-6 - :button_link: basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: 2: Enable two-way communication - :description: Enable two-way communication between the dash app and a Lightning App. - :col_css: col-md-6 - :button_link: intermediate.html - :height: 150 - :tag: [docs coming soon] - -.. raw:: html - -
-
- ----- - -******** -Examples -******** -Here are a few example apps that use a Dash web UI. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/dash/intermediate.rst b/docs/source-app/workflows/add_web_ui/dash/intermediate.rst deleted file mode 100644 index 5c5da1a19403e..0000000000000 --- a/docs/source-app/workflows/add_web_ui/dash/intermediate.rst +++ /dev/null @@ -1,42 +0,0 @@ -##################################### -Add a web UI with Dash (intermediate) -##################################### -**Audience:** Users who want to communicate between the Lightning App and Dash. - -**Prereqs:** Must have read the :doc:`dash basic ` guide. - ----- - -******************************* -Interact with the App from Dash -******************************* - -In the example below, every time you change the select year on the dashboard, this is directly communicated to the flow -and another work process the associated data frame with the provided year. - -.. literalinclude:: intermediate_plot.py - -Here is how the app looks like once running: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/dash_plot.gif - ----- - -*********************************** -Interact with Dash from a component -*********************************** - -In the example below, when you click the toggle, the state of the work appears. - -Install the following libraries if you want to run the app. - -```bash -pip install dash_daq dash_renderjson -``` - -.. literalinclude:: intermediate_state.py - - -Here is how the app looks like once running: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/dash_state.gif diff --git a/docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py b/docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py deleted file mode 100644 index 95a3edb8f99d4..0000000000000 --- a/docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import Optional - -import pandas as pd -import plotly.express as px -from dash import Dash, dcc, html, Input, Output - -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.storage import Payload - - -class LitDash(LightningWork): - def __init__(self): - super().__init__(parallel=True) - self.df = None - self.selected_year = None - - def run(self): - df = pd.read_csv("https://raw.githubusercontent.com/plotly/datasets/master/gapminderDataFiveYear.csv") - self.df = Payload(df) - - dash_app = Dash(__name__) - - dash_app.layout = html.Div( - [ - dcc.Graph(id="graph-with-slider"), - dcc.Slider( - df["year"].min(), - df["year"].max(), - step=None, - value=df["year"].min(), - marks={str(year): str(year) for year in df["year"].unique()}, - id="year-slider", - ), - ] - ) - - @dash_app.callback(Output("graph-with-slider", "figure"), Input("year-slider", "value")) - def update_figure(selected_year): - self.selected_year = selected_year - filtered_df = df[df.year == selected_year] - - fig = px.scatter( - filtered_df, - x="gdpPercap", - y="lifeExp", - size="pop", - color="continent", - hover_name="country", - log_x=True, - size_max=55, - ) - - fig.update_layout(transition_duration=500) - - return fig - - dash_app.run_server(host=self.host, port=self.port) - - -class Processor(LightningWork): - def run(self, df: Payload, selected_year: Optional[str]): - if selected_year: - df = df.value - filtered_df = df[df.year == selected_year] - print(f"[PROCESSOR|selected_year={selected_year}]") - print(filtered_df) - - -class LitApp(LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash() - self.processor = Processor(parallel=True) - - def run(self): - self.lit_dash.run() - - # Launch some processing based on the Dash Dashboard. - self.processor.run(self.lit_dash.df, self.lit_dash.selected_year) - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - -app = LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/dash/intermediate_state.py b/docs/source-app/workflows/add_web_ui/dash/intermediate_state.py deleted file mode 100644 index d30bdc8c02c25..0000000000000 --- a/docs/source-app/workflows/add_web_ui/dash/intermediate_state.py +++ /dev/null @@ -1,39 +0,0 @@ -import dash -import dash_daq as daq -import dash_renderjson -from dash import html, Input, Output - -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.utilities.state import AppState - - -class LitDash(LightningWork): - def run(self): - dash_app = dash.Dash(__name__) - - dash_app.layout = html.Div([daq.ToggleSwitch(id="my-toggle-switch", value=False), html.Div(id="output")]) - - @dash_app.callback(Output("output", "children"), [Input("my-toggle-switch", "value")]) - def display_output(value): - if value: - state = AppState() - state._request_state() - return dash_renderjson.DashRenderjson(id="input", data=state._state, max_depth=-1, invert_theme=True) - - dash_app.run_server(host=self.host, port=self.port) - - -class LitApp(LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash(parallel=True) - - def run(self): - self.lit_dash.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - -app = LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/example_app.rst b/docs/source-app/workflows/add_web_ui/example_app.rst deleted file mode 100644 index e5d2cbbd3f8d4..0000000000000 --- a/docs/source-app/workflows/add_web_ui/example_app.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -########### -Example App -########### - -This is an example app that needs to be built for this part of the docs. diff --git a/docs/source-app/workflows/add_web_ui/glossary_front_end.rst b/docs/source-app/workflows/add_web_ui/glossary_front_end.rst deleted file mode 100644 index ce51ef12aa77b..0000000000000 --- a/docs/source-app/workflows/add_web_ui/glossary_front_end.rst +++ /dev/null @@ -1,9 +0,0 @@ -######## -Frontend -######## -Web pages visible to users are also known as **front-ends**. Lightning Apps can have multiple -types of Frontends. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/add_web_ui/glossary_ui.rst b/docs/source-app/workflows/add_web_ui/glossary_ui.rst deleted file mode 100644 index bc9e4f529e3b6..0000000000000 --- a/docs/source-app/workflows/add_web_ui/glossary_ui.rst +++ /dev/null @@ -1,9 +0,0 @@ -################### -UI (User Interface) -################### -We use (UI) as short for a **web page** with interactions. Lightning Apps can have multiple -types of UIs. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/add_web_ui/gradio/basic.rst b/docs/source-app/workflows/add_web_ui/gradio/basic.rst deleted file mode 100644 index 4f5ab87b8b8d4..0000000000000 --- a/docs/source-app/workflows/add_web_ui/gradio/basic.rst +++ /dev/null @@ -1,217 +0,0 @@ -################################ -Add a web UI with Gradio (basic) -################################ -**Audience:** Users who want to add a web UI written with Python. - -**Prereqs:** Basic python knowledge. - ----- - -*************** -What is Gradio? -*************** -Gradio is a Python library that automatically generates a web interface to demo a machine learning model. - ----- - -***************** -Install gradio -***************** -First, install gradio. - -.. code:: bash - - pip install gradio - ----- - -************************** -Create the gradio demo app -************************** -To explain how to use Gradio with Lightning, let's replicate the |gradio_link|. - -.. |gradio_link| raw:: html - - example running here - -In the next few sections we'll build an app step-by-step. -First **create a file named app.py** with the app content: - -.. code:: python - - import lightning as L - from lightning.app.components import ServeGradio - import gradio as gr - - class LitGradio(ServeGradio): - - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') - examples = [["hello lightning"]] - - def predict(self, input_text): - return self.model(input_text) - - def build_model(self): - fake_model = lambda x: f"hello {x}" - return fake_model - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_gradio = LitGradio() - - def run(self): - self.lit_gradio.run() - - def configure_layout(self): - return [{"name": "home", "content": self.lit_gradio}] - - app = L.LightningApp(RootFlow()) - -add "gradio" to a requirements.txt file: - -.. code:: bash - - echo 'gradio' >> requirements.txt - -this is a best practice to make apps reproducible. - ----- - -*********** -Run the app -*********** -Run the app locally to see it! - -.. code:: python - - lightning run app app.py - -Now run it on the cloud as well: - -.. code:: python - - lightning run app app.py --cloud - ----- - -************************ -Step-by-step walkthrough -************************ -In this section, we explain each part of this code in detail. - ----- - -Create a Gradio component -^^^^^^^^^^^^^^^^^^^^^^^^^ -To create a Gradio component, simply take any Gradio app and subclass it from ``ServeGradio``. -If you haven't created a Gradio demo, you have to implement the following elements: - -1. Input which is text. -2. Output which is text. -3. A build_model function. -4. A predict function. - -| - -Here's an example: - -.. code:: python - :emphasize-lines: 4 - - from lightning.app.components import ServeGradio - import gradio as gr - - class LitGradio(ServeGradio): - - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') - - def predict(self, input_text): - return self.model(input_text) - - def build_model(self): - fake_model = lambda x: f"hello {x}" - return fake_model - -This fake model simply concatenates 2 strings. - ----- - -Route the UI in the root component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now, tell the Root component in which tab to render this component's UI. -In this case, we render the ``LitGradio`` UI in the ``home`` tab of the application. - -.. code:: python - :emphasize-lines: 21, 27 - - import lightning as L - from lightning.app.components import ServeGradio - import gradio as gr - - class LitGradio(ServeGradio): - - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') - examples = [["hello lightning"]] - - def predict(self, input_text): - return self.model(input_text) - - def build_model(self): - fake_model = lambda x: f"hello {x}" - return fake_model - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_gradio = LitGradio() - - def run(self): - self.lit_gradio.run() - - def configure_layout(self): - return [{"name": "home", "content": self.lit_gradio}] - - app = L.LightningApp(RootFlow()) - ----- - -Call run -^^^^^^^^ -Finally, don't forget to call run inside the Root Flow to serve the Gradio app. - -.. code:: python - :emphasize-lines: 24 - - import lightning as L - from lightning.app.components import ServeGradio - import gradio as gr - - class LitGradio(ServeGradio): - - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') - examples = [["hello lightning"]] - - def predict(self, input_text): - return self.model(input_text) - - def build_model(self): - fake_model = lambda x: f"hello {x}" - return fake_model - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_gradio = LitGradio() - - def run(self): - self.lit_gradio.run() - - def configure_layout(self): - return [{"name": "home", "content": self.lit_gradio}] - - app = L.LightningApp(RootFlow()) diff --git a/docs/source-app/workflows/add_web_ui/gradio/index.rst b/docs/source-app/workflows/add_web_ui/gradio/index.rst deleted file mode 100644 index 740ae93aae0c4..0000000000000 --- a/docs/source-app/workflows/add_web_ui/gradio/index.rst +++ /dev/null @@ -1,84 +0,0 @@ -:orphan: - -.. toctree:: - :maxdepth: 1 - :hidden: - - basic - intermediate - -######################## -Add a web UI with Gradio -######################## - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1: Connect Gradio - :description: Learn how to connect Gradio to a Lightning Component. - :col_css: col-md-6 - :button_link: basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: 2: Enable two-way communication - :description: Enable two-way communication between Gradio and a Lightning App. - :col_css: col-md-6 - :button_link: intermediate.html - :height: 150 - :tag: [documentation coming soon] - -.. raw:: html - -
-
- ----- - -******** -Examples -******** -Here are a few example apps that use a Gradio web UI. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst b/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst deleted file mode 100644 index bb20d566243b1..0000000000000 --- a/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst +++ /dev/null @@ -1,21 +0,0 @@ -####################################### -Add a web UI with Gradio (intermediate) -####################################### - -.. note:: documentation coming soon. - - -************************************* -Interact with a component from the UI -************************************* - -.. warning:: is there such a thing for this with gradio? - - ----- - -************************************* -Interact with the UI from a component -************************************* - -.. warning:: is there such a thing for this with gradio? diff --git a/docs/source-app/workflows/add_web_ui/html/basic.rst b/docs/source-app/workflows/add_web_ui/html/basic.rst deleted file mode 100644 index cb9bb5293f7c9..0000000000000 --- a/docs/source-app/workflows/add_web_ui/html/basic.rst +++ /dev/null @@ -1,166 +0,0 @@ -############################## -Add a web UI with HTML (basic) -############################## -**Audience:** Users who want to add a web UI written in HTMlapp. - -**Prereqs:** Basic html knowledge. - ----- - -************* -What is HTML? -************* -HyperText Markup Language (HTML) is the Language used to create web pages. Use HTML for simple -web user interfaces that tend to be more static. - -For reactive web applications, we recommend using: React.js, Angular.js or Vue.js - ----- - -******************* -Create an HTML page -******************* -The first step is to create an HTML file named **index.html**: - -.. code:: html - - - - - - -

Hello World

- - - ----- - -************************ -Create the HTML demo app -************************ - -.. - To explain how to use html with Lightning, let's replicate the |html_app_link|. - - .. |html_app_link| raw:: html - - example running here - -In the next few sections we'll build an app step-by-step. -First **create a file named app.py** with the app content (in the same folder as index.html): - -.. code:: bash - - # app.py - import lightning as L - import lightning.app.frontend as frontend - - - class HelloComponent(L.LightningFlow): - def configure_layout(self): - return frontend.StaticWebFrontend(serve_dir='.') - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.hello_component = HelloComponent() - - def run(self): - self.hello_component.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.hello_component} - return tab1 - - - app = L.LightningApp(LitApp()) - ----- - -*********** -Run the app -*********** -Run the app locally to see it! - -.. code:: python - - lightning run app app.py - -Now run it on the cloud as well: - -.. code:: python - - lightning run app app.py --cloud - ----- - -************************ -Step-by-step walkthrough -************************ -In this section, we explain each part of this code in detail. - ----- - -Enable an HTML UI for the component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Give the component an HTML UI, by returning a ``StaticWebFrontend`` object from the ``configure_layout`` method: - -.. code:: bash - :emphasize-lines: 6,7 - - # app.py - import lightning as L - import lightning.app.frontend as frontend - - class HelloComponent(L.LightningFlow): - def configure_layout(self): - return frontend.StaticWebFrontend(serve_dir='.') - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.hello_component = HelloComponent() - - def run(self): - self.hello_component.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.hello_component} - return tab1 - - app = L.LightningApp(LitApp()) - -The folder path given in ``StaticWebFrontend(serve_dir=)`` must point to a folder with an ``index.html`` page. - ----- - -Route the UI in the root component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The final step, is to tell the Root component in which tab to render this component's UI. -In this case, we render the ``HelloComponent`` UI in the ``home`` tab of the application. - -.. code:: python - :emphasize-lines: 18, 19 - - # app.py - import lightning as L - import lightning.app.frontend as frontend - - class HelloComponent(L.LightningFlow): - def configure_layout(self): - return frontend.StaticWebFrontend(serve_dir='.') - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.hello_component = HelloComponent() - - def run(self): - self.hello_component.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.hello_component} - return tab1 - - app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/html/index.rst b/docs/source-app/workflows/add_web_ui/html/index.rst deleted file mode 100644 index 0eae9309877a7..0000000000000 --- a/docs/source-app/workflows/add_web_ui/html/index.rst +++ /dev/null @@ -1,87 +0,0 @@ -:orphan: - -.. toctree:: - :maxdepth: 1 - :hidden: - - basic - intermediate - -###################### -Add a web UI with HTML -###################### -**Audience:** Users who want to add a web UI using plain html. - -**Prereqs:** Basic html knowledge. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1: Connect HTML - :description: Learn how to connect an HTML app. - :col_css: col-md-6 - :button_link: basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: 2: Enable two-way communication - :description: Enable two-way communication between HTML and a Lightning App. - :col_css: col-md-6 - :button_link: intermediate.html - :height: 150 - :tag: [docs coming soon] - -.. raw:: html - -
-
- ----- - -******** -Examples -******** -Here are a few example apps that use an HTML web UI. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/html/intermediate.rst b/docs/source-app/workflows/add_web_ui/html/intermediate.rst deleted file mode 100644 index 17b6b585c0173..0000000000000 --- a/docs/source-app/workflows/add_web_ui/html/intermediate.rst +++ /dev/null @@ -1,20 +0,0 @@ -##################################### -Add a web UI with HTML (intermediate) -##################################### -**Audience:** Users who want to add a web UI using plain html. - -**Prereqs:** Must have read the :doc:`html basic ` guide. - ----- - -******************************* -Interact with the App from HTML -******************************* -.. note:: documentation in progress - ----- - -*********************************** -Interact with HTML from a component -*********************************** -.. note:: documentation in progress diff --git a/docs/source-app/workflows/add_web_ui/index.rst b/docs/source-app/workflows/add_web_ui/index.rst deleted file mode 100644 index 79c0f16d66409..0000000000000 --- a/docs/source-app/workflows/add_web_ui/index.rst +++ /dev/null @@ -1,10 +0,0 @@ - -############################# -Add a web user interface (UI) -############################# - -**Audience:** Users who want to add a UI to their Lightning Apps - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/add_web_ui/index_content.rst b/docs/source-app/workflows/add_web_ui/index_content.rst deleted file mode 100644 index f3d516c5af546..0000000000000 --- a/docs/source-app/workflows/add_web_ui/index_content.rst +++ /dev/null @@ -1,121 +0,0 @@ -************************************* -Web UIs for non Javascript Developers -************************************* - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Dash - :description: Learn how to add a web UI built in Python with Dash. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/dash/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Gradio - :description: Learn how to add a web UI built in Python with Gradio. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/gradio/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Panel - :description: Learn how to add a web UI built in Python with Panel. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/panel/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Jupyter Notebook - :description: Learn how to enable a web UI that is a Jupyter Notebook. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/jupyter_basic.html - :height: 150 - :tag: [docs coming soon] - -.. displayitem:: - :header: Streamlit - :description: Learn how to add a web UI built in Python with Streamlit. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/streamlit/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: JustPy - :description: Learn how to add a web UI built in Python with JustPy. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/justpy/index.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
- ----- - -********************************* -Web UIs for Javascript Developers -********************************* - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Any javascript framework - :description: Learn how to link up any javascript framework to a Lightning app. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/integrate_any_javascript_framework.html - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Angular.js - :description: Learn how to add a web UI built in Javascript with Angular.js - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/angular_js_intermediate.html - :height: 150 - :tag: [Docs coming soon] - -.. displayitem:: - :header: HTML - :description: Learn how to add a web UI built with html. - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/html/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: React.js - :description: Learn how to add a web UI built in Javascript with React.js - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/react/index.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Vue.js - :description: Learn how to add a web UI built in Javascript with Vue.js - :col_css: col-md-4 - :button_link: ../../workflows/add_web_ui/vue_js_intermediate.html - :height: 150 - :tag: [Docs coming soon] - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst b/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst deleted file mode 100644 index 66625289a6e49..0000000000000 --- a/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst +++ /dev/null @@ -1,164 +0,0 @@ -:orphan: - -################################## -Integrate any javascript framework -################################## -**Audience:** Advanced web developers with complex apps that may not have been covered by the other tutorials - -**Pre-requisites:** Intermediate knowledge of html and javascript - ----- - -************************ -Import LightningState.js -************************ -To connect any javascript framework, import the `LightningState.js `_ library. -LightningState.js enables two-way communication between a javascript framework and a Lightning app. - -To import this library, add this to your html: - -.. code:: html - - - -Once it's imported, use it inside your app, this example uses it inside a React App: - -.. code-block:: - :emphasize-lines: 1, 5 - - import { useLightningState } from "./hooks/useLightningState"; - import cloneDeep from "lodash/cloneDeep"; - - function App() { - const { lightningState, updateLightningState } = useLightningState(); - - const modify_and_send_back_the_state = async (event: ChangeEvent) => { - if (lightningState) { - const newLightningState = cloneDeep(lightningState); - // Update the state and send it back. - newLightningState.flows.counter += 1 - - updateLightningState(newLightningState); - } - }; - - return ( -
-
- ); - } - - export default App; - ----- - -************************ -Update the Lightning app -************************ -Use `updateLightningState` to update the lightning app. Here we update a variable called counter. - -.. code-block:: - :emphasize-lines: 11 - - import { useLightningState } from "./hooks/useLightningState"; - import cloneDeep from "lodash/cloneDeep"; - - function App() { - const { lightningState, updateLightningState } = useLightningState(); - - const modify_and_send_back_the_state = async (event: ChangeEvent) => { - if (lightningState) { - const newLightningState = cloneDeep(lightningState); - // Update the state and send it back. - newLightningState.flows.counter += 1 - - updateLightningState(newLightningState); - } - }; - - return ( -
-
- ); - } - - export default App; - ----- - -************************************** -Receive updates from the Lightning app -************************************** -Whenever a variable in the Lightning app changes, the javascript app will receive those values via `lightningState`. - -Extract any variable from the state and update the javascript app: - -.. code-block:: - :emphasize-lines: 5 - - import { useLightningState } from "./hooks/useLightningState"; - import cloneDeep from "lodash/cloneDeep"; - - function App() { - const { lightningState, updateLightningState } = useLightningState(); - - const modify_and_send_back_the_state = async (event: ChangeEvent) => { - if (lightningState) { - const newLightningState = cloneDeep(lightningState); - // Update the state and send it back. - newLightningState.flows.counter += 1 - - updateLightningState(newLightningState); - } - }; - - return ( -
-
- ); - } - - export default App; - ----- - -******** -Examples -******** - -See this in action in these examples: - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: React.js - :description: Explore how React.js uses lightningState.js - :col_css: col-md-4 - :button_link: react/communicate_between_react_and_lightning.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/jupyter_basic.rst b/docs/source-app/workflows/add_web_ui/jupyter_basic.rst deleted file mode 100644 index 61f58ab40670d..0000000000000 --- a/docs/source-app/workflows/add_web_ui/jupyter_basic.rst +++ /dev/null @@ -1,70 +0,0 @@ -:orphan: - -##################################### -Add a Jupyter Notebook web UI (basic) -##################################### -**Audience:** Users who want to enable a Jupyter notebook UI. - -**Prereqs:** Basic python knowledge. - -TODO - ----- - -*************************** -What is a Jupyter Notebook? -*************************** - -TODO - ----- - -******************* -Install Jupyter Lab -******************* - -First, install Jupyter Lab. - -.. code:: bash - - pip install jupyterlab - ----- - -******** -Examples -******** -Here are a few example apps that use Jupyter Lab. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: TODO - :col_css: col-md-4 - :button_link: angular_js_intermediate.html - :height: 150 - -.. displayitem:: - :header: Example 2 - :description: TODO - :col_css: col-md-4 - :button_link: angular_js_intermediate.html - :height: 150 - -.. displayitem:: - :header: Example 3 - :description: TODO - :col_css: col-md-4 - :button_link: angular_js_intermediate.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/justpy/index.rst b/docs/source-app/workflows/add_web_ui/justpy/index.rst deleted file mode 100644 index 1626930c13d82..0000000000000 --- a/docs/source-app/workflows/add_web_ui/justpy/index.rst +++ /dev/null @@ -1,92 +0,0 @@ -:orphan: - -######################## -Add a web UI with JustPy -######################## - - -****** -JustPy -****** - -The `JustPy `_ framework is an object oriented high-level Python Web Framework that requires no JavaScript programming, while at the same time providing the full flexibility of a frontend framework. - -Additionally, it provides a higher level API called `Quasar `_ with stylized components. - - -You can install ``justpy`` from PyPi. - -.. code-block:: - - pip install justpy - -******* -Example -******* - - -In the following example, we are creating a simple UI with 2 buttons. -When clicking the first button, the flow state ``counter`` is incremented and re-rendered on the UI. - - -First of all, you would need to import the ``JustPyFrontend`` and return it from the ``configure_layout`` hook of the flow. - -.. code-block:: - - from typing import Callable - - from lightning import LightningApp, LightningFlow - from lightning.app.frontend import JustPyFrontend - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.counter = 0 - - def run(self): - print(self.counter) - - def configure_layout(self): - return JustPyFrontend(render_fn=render_fn) - -Secondly, you would need to implement a ``render_fn`` that takes as input a ``get_state`` function and return a function. - - -.. code-block:: - - def render_fn(get_state: Callable) -> Callable: - import justpy as jp - - def webpage(): - wp = jp.QuasarPage(dark=True) - # the `a=wp` argument adds the div to the web page - d = jp.Div(classes="q-pa-md q-gutter-sm", a=wp) - container = jp.QBtn(color="primary", text="Counter: 0") - - async def click(*_): - state = get_state() - state.counter += 1 - container.text = f"Counter: {state.counter}" - - button = jp.QBtn(color="primary", text="Click Me!", click=click) - - d.add(button) - d.add(container) - - return wp - - return webpage - - -Finally, you can wrap your flow in a LightningAp. - -.. code-block:: - - app = LightningApp(Flow()) - -Now, you can run the Lightning App with: - -.. code-block:: - - lightning_app run app app.py diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst deleted file mode 100644 index b033312f6f0d1..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ /dev/null @@ -1,369 +0,0 @@ -:orphan: - -############################### -Add a web UI with Panel (basic) -############################### - -**Audience:** Users who want to add a web UI written with Python and Panel. - -**Prereqs:** Basic Python knowledge. - ----- - -************** -What is Panel? -************** - -`Panel`_ and the `HoloViz`_ ecosystem provide unique and powerful -features such as big data visualization using `DataShader`_, easy cross filtering -using `HoloViews`_, streaming and much more. - -* Panel is highly flexible and ties into the PyData and Jupyter ecosystems as you can develop in notebooks and use ipywidgets. You can also develop in .py files. - -* Panel is one of the most popular data app frameworks in Python with `more than 400.000 downloads a month `_. It's especially popular in the scientific community. - -* Panel is used, for example, by Rapids to power `CuxFilter`_, a CuDF based big data visualization framework. - -* Panel can be deployed on your favorite server or cloud including `Lightning`_. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-intro.gif - :alt: Example Panel App - - Example Panel App - -Panel is **particularly well suited for Lightning Apps** that need to display live progress. This is because the Panel server can react -to state changes and asynchronously push messages from the server to the client using web socket communication. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-streaming-intro.gif - :alt: Example Panel Streaming App - - Example Panel Streaming App - -Install Panel with: - -.. code:: bash - - pip install panel - ----- - -********************* -Run a basic Panel App -********************* - -In the next few sections, we'll build an App step-by-step. - -First, create a file named ``app_panel.py`` with the App content: - -.. code:: python - - # app_panel.py - - import panel as pn - - pn.panel("Hello **Panel ⚡** World").servable() - -Then, create a file named ``app.py`` with the following App content: - -.. code:: python - - # app.py - - import lightning as L - from lightning.app.frontend import PanelFrontend - - - class LitPanel(L.LightningFlow): - - def configure_layout(self): - return PanelFrontend("app_panel.py") - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self): - self.lit_panel.run() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - - app = L.LightningApp(LitApp()) - -Finally, add ``panel`` to your ``requirements.txt`` file: - -.. code:: bash - - echo 'panel' >> requirements.txt - -.. note:: This is a best practice to make Apps reproducible. - ----- - -*********** -Run the App -*********** - -Run the App locally: - -.. code:: bash - - lightning_app run app app.py - -The App should look like this: - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-basic.png - :alt: Basic Panel Lightning App - - Basic Panel Lightning App - -Now, run it on the cloud: - -.. code:: bash - - lightning_app run app app.py --cloud - ----- - -************************* -Step-by-step walk-through -************************* - -In this section, we explain each part of the code in detail. - ----- - -0. Define a Panel app -^^^^^^^^^^^^^^^^^^^^^ - -First, find the Panel app you want to integrate. In this example, that app looks like: - -.. code:: python - - import panel as pn - - pn.panel("Hello **Panel ⚡** World").servable() - -Refer to the `Panel documentation `_ and `awesome-panel `_ for more complex examples. - ----- - -1. Add Panel to a Component -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Link this app to the Lightning App by using the ``PanelFrontend`` class which needs to be returned from -the ``configure_layout`` method of the Lightning Component you want to connect to Panel. - -.. code:: python - :emphasize-lines: 7-10 - - import lightning as L - from lightning.app.frontend import PanelFrontend - - - class LitPanel(L.LightningFlow): - - def configure_layout(self): - return PanelFrontend("app_panel.py") - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self): - self.lit_panel.run() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - - app = L.LightningApp(LitApp()) - -The argument of the ``PanelFrontend`` class, points to the script, notebook, or function that -runs your Panel app. - ----- - -2. Route the UI in the root component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The second step, is to tell the Root component in which tab to render this component's UI. -In this case, we render the ``LitPanel`` UI in the ``home`` tab of the app. - -.. code:: python - :emphasize-lines: 19-20 - - import lightning as L - from lightning.app.frontend import PanelFrontend - - - class LitPanel(L.LightningFlow): - - def configure_layout(self): - return PanelFrontend("app_panel.py") - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self): - self.lit_panel.run() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - app = L.LightningApp(LitApp()) - ----- - -************* -Tips & Tricks -************* - -0. Use autoreload while developing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To speed up your development workflow, you can run your Lightning App with Panel **autoreload** by -setting the environment variable ``PANEL_AUTORELOAD`` to ``yes``. - -Try running the following: - -.. code-block:: - - PANEL_AUTORELOAD=yes lightning run app app.py - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-autoreload.gif - :alt: Basic Panel Lightning App with autoreload - - Basic Panel Lightning App with autoreload - -1. Theme your App -^^^^^^^^^^^^^^^^^ - -To theme your App you, can use the Lightning accent color ``#792EE5`` with the `FastListTemplate`_. - -Try replacing the contents of ``app_panel.py`` with the following: - -.. code:: bash - - # app_panel.py - - import panel as pn - import plotly.express as px - - ACCENT = "#792EE5" - - pn.extension("plotly", sizing_mode="stretch_width", template="fast") - pn.state.template.param.update( - title="⚡ Hello Panel + Lightning ⚡", accent_base_color=ACCENT, header_background=ACCENT - ) - - pn.config.raw_css.append( - """ - .bk-root:first-of-type { - height: calc( 100vh - 200px ) !important; - } - """ - ) - - - def get_panel_theme(): - """Returns 'default' or 'dark'""" - return pn.state.session_args.get("theme", [b"default"])[0].decode() - - - def get_plotly_template(): - if get_panel_theme() == "dark": - return "plotly_dark" - return "plotly_white" - - - def get_plot(length=5): - xseries = [index for index in range(length + 1)] - yseries = [x**2 for x in xseries] - fig = px.line( - x=xseries, - y=yseries, - template=get_plotly_template(), - color_discrete_sequence=[ACCENT], - range_x=(0, 10), - markers=True, - ) - fig.layout.autosize = True - return fig - - - length = pn.widgets.IntSlider(value=5, start=1, end=10, name="Length") - dynamic_plot = pn.panel( - pn.bind(get_plot, length=length), sizing_mode="stretch_both", config={"responsive": True} - ) - pn.Column(length, dynamic_plot).servable() - - -Install some additional libraries and remember to add the dependencies to the ``requirements.txt`` file: - - -.. code:: bash - - echo 'plotly' >> requirements.txt - echo 'pandas' >> requirements.txt - -Finally run the App - -.. code:: bash - - lightning_app run app app.py - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-theme.gif - :alt: Basic Panel Plotly Lightning App with theming - - Basic Panel Plotly Lightning App with theming - -.. _Panel: https://panel.holoviz.org/ -.. _FastListTemplate: https://panel.holoviz.org/reference/templates/FastListTemplate.html#templates-gallery-fastlisttemplate -.. _HoloViz: https://holoviz.org/ -.. _DataShader: https://datashader.org/ -.. _HoloViews: https://holoviews.org/ -.. _Lightning: https://lightning.ai/ -.. _CuxFilter: https://github.com/rapidsai/cuxfilter -.. _AwesomePanel: https://github.com/awesome-panel/awesome-panel - - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 2: Enable two-way communication - :description: Enable two-way communication between Panel and a Lightning App. - :col_css: col-md-6 - :button_link: intermediate.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Add a web user interface (UI) - :description: Users who want to add a UI to their Lightning Apps - :col_css: col-md-6 - :button_link: ../index.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/panel/index.rst b/docs/source-app/workflows/add_web_ui/panel/index.rst deleted file mode 100644 index 0d48a1dc9f7ea..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/index.rst +++ /dev/null @@ -1,85 +0,0 @@ -:orphan: - -.. toctree:: - :maxdepth: 1 - :hidden: - - basic - intermediate - -####################### -Add a web UI with Panel -####################### - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1: Connect Panel - :description: Learn how to connect Panel to a Lightning Component. - :col_css: col-md-6 - :button_link: basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: 2: Enable two-way communication - :description: Enable two-way communication between Panel and a Lightning App. - :col_css: col-md-6 - :button_link: intermediate.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
- ----- - -******** -Examples -******** - -Here are a few example apps that use a Panel web UI. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst deleted file mode 100644 index 20cf141248739..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ /dev/null @@ -1,210 +0,0 @@ -:orphan: - -###################################### -Add a web UI with Panel (intermediate) -###################################### - -**Audience:** Users who want to communicate between the Lightning App and Panel. - -**Prereqs:** Must have read the :doc:`Panel basic ` guide. - ----- - -************************************** -Interact with the Component from Panel -************************************** - -The ``PanelFrontend`` enables user interactions with the Lightning App using widgets. -You can modify the state variables of a Lightning Component using the ``AppStateWatcher``. - -For example, here we increase the ``count`` variable of the Lightning Component every time a user -presses a button: - -.. code:: python - - # app_panel.py - - import panel as pn - from lightning.app.frontend import AppStateWatcher - - pn.extension(sizing_mode="stretch_width") - - app = AppStateWatcher() - - submit_button = pn.widgets.Button(name="submit") - - @pn.depends(submit_button, watch=True) - def submit(_): - app.state.count += 1 - - @pn.depends(app.param.state) - def current_count(_): - return f"current count: {app.state.count}" - - pn.Column( - submit_button, - current_count, - ).servable() - - - -.. code:: python - - # app.py - - import lightning as L - from lightning.app.frontend import PanelFrontend - - class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self.count = 0 - self.last_count = 0 - - def run(self): - if self.count != self.last_count: - self.last_count = self.count - print("Count changed to: ", self.count) - - def configure_layout(self): - return PanelFrontend("app_panel.py") - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self): - self.lit_panel.run() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - - app = L.LightningApp(LitApp()) - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-counter-from-frontend.gif - :alt: Panel Lightning App updating a counter from the frontend - - Panel Lightning App updating a counter from the frontend - ----- - -************************************ -Interact with Panel from a Component -************************************ - -To update the `PanelFrontend` from any Lightning Component, update the property in the Component. -Make sure to call the ``run`` method from the parent component. - -In this example, we update the ``count`` value of the Component: - -.. code:: python - - # app_panel.py - - import panel as pn - from lightning.app.frontend import AppStateWatcher - - app = AppStateWatcher() - - pn.extension(sizing_mode="stretch_width") - - def counter(state): - return f"Counter: {state.count}" - - last_update = pn.bind(counter, app.param.state) - - pn.panel(last_update).servable() - -.. code:: python - - # app.py - - from datetime import datetime as dt - from lightning.app.frontend import PanelFrontend - - import lightning as L - - - class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self.count = 0 - self._last_update = dt.now() - - def run(self): - now = dt.now() - if (now - self._last_update).microseconds >= 250: - self.count += 1 - self._last_update = now - print("Counter changed to: ", self.count) - - def configure_layout(self): - return PanelFrontend("app_panel.py") - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self): - self.lit_panel.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_panel} - return tab1 - - app = L.LightningApp(LitApp()) - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-counter-from-component.gif - :alt: Panel Lightning App updating a counter from the component - - Panel Lightning App updating a counter from the Component - ----- - -************* -Tips & Tricks -************* - -* Caching: Panel provides the easy to use ``pn.state.cache`` memory based, ``dict`` caching. If you are looking for something persistent try `DiskCache `_ its really powerful and simple to use. You can use it to communicate large amounts of data between the components and frontend(s). - -* Notifications: Panel provides easy to use `notifications `_. You can for example use them to provide notifications about runs starting or ending. - -* Tabulator Table: Panel provides the `Tabulator table `_ which features expandable rows. The table is useful to provide for example an overview of you runs. But you can dig into the details by clicking and expanding the row. - -* Task Scheduling: Panel provides easy to use `task scheduling `_. You can use this to for example read and display files created by your components on a scheduled basis. - -* Terminal: Panel provides the `Xterm.js terminal `_ which can be used to display live logs from your components and allow you to provide a terminal interface to your component. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/panel-lightning-github-runner.gif - :alt: Panel Lightning App running models on github - - Panel Lightning App running models on GitHub - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add a web user interface (UI) - :description: Users who want to add a UI to their Lightning Apps - :col_css: col-md-6 - :button_link: ../index.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst b/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst deleted file mode 100644 index ef836e7e4b0d1..0000000000000 --- a/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst +++ /dev/null @@ -1,58 +0,0 @@ -####################################### -Communicate Between React and Lightning -####################################### -**Audience:** Anyone who wants to add a web user interface (UI) written in react to their app. - -**pre-requisites:** Make sure you've already connected the React and Lightning app. - -**Difficulty level:** intermediate. - ----- - -************ -Example code -************ -To illustrate how to communicate between a React app and a lightning App, we'll be using the `example_app.py` file -which :doc:`lightning init react-ui ` created: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - -and the App.tsx file also created by :doc:`lightning init react-ui `: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/ui/src/App.tsx - ----- - -****************************** -Update React --> Lightning app -****************************** -To change the Lightning app from the React app, use `updateLightningState`. - -In this example, when you press **Start printing** in the React UI, it toggles -the `react_ui.vars.should_print`: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/ui/src/App.tsx - :emphasize-lines: 20, 21, 23 - -By changing that variable in the Lightning app state, it sets **react_ui.should_print** to True, which enables the -Lightning app to print: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - :emphasize-lines: 10, 22 - ----- - -****************************** -Update React <-- Lightning app -****************************** -To change the React app from the Lightning app, use the values from the `lightningState`. - -In this example, when the ``react_ui.counter`` increaes in the Lightning app: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - :emphasize-lines: 18, 24 - -The React UI updates the text on the screen to reflect the count - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/ui/src/App.tsx - :emphasize-lines: 15 diff --git a/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst b/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst deleted file mode 100644 index bacdd3d299b05..0000000000000 --- a/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst +++ /dev/null @@ -1,107 +0,0 @@ -################################ -Connect React to a Lightning app -################################ -**Audience:** Users who already have a react app and want to connect it to a Lightning app. - -**pre-requisites:** Make sure you already have a react app you want to connect. - -**Difficulty level:** intermediate. - ----- - -************ -Example code -************ -To illustrate how to connect a React app and a lightning App, we'll be using the `example_app.py` file -which :doc:`lightning_app init react-ui ` created: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - -and the App.tsx file also created by :doc:`lightning_app init react-ui `: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/ui/src/App.tsx - ----- - -************************************* -Connect the component to the react UI -************************************* -The first step is to connect the dist folder of the react app using `StaticWebFrontend`: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - :emphasize-lines: 13 - -the dist folder must contain an index.html file which is generated by the compilating command `yarn build` which -we'll explore later. - ----- - -********************************** -Connect component to the root flow -********************************** -Next, connect your component to the root flow. Display the react app on the tab of your choice -using `configure_layout`: - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - :emphasize-lines: 19, 27 - ----- - -********************************* -Connect React and Lightning state -********************************* -At this point, the React app will render in the Lightning app. Test it out! - -.. code:: bash - - lightning_app run app example_app.py - -However, to make powerful React+Lightning apps, you must also connect the Lightning App state to the react app. -These lines enable two-way communication between the react app and the Lightning app. - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/ui/src/App.tsx - :emphasize-lines: 10, 13 - ----- - -**************** -Component vs App -**************** -Notice that in this guide, we connected a single react app to a single component. - -.. literalinclude:: ../../../../../src/lightning/app/cli/react-ui-template/example_app.py - :emphasize-lines: 6-13 - -You can use this single react app for the FULL Lightning app, or you can specify a React app for EACH component. - -.. code:: python - :emphasize-lines: 5, 9, 18-20 - - import lightning as L - - - class ComponentA(L.LightningFlow): - def configure_layout(self): - return L.app.frontend.StaticWebFrontend(Path(__file__).parent / "react_app_1/dist") - - - class ComponentB(L.LightningFlow): - def configure_layout(self): - return L.app.frontend.StaticWebFrontend(Path(__file__).parent / "react_app_2/dist") - - - class HelloLitReact(L.LightningFlow): - def __init__(self): - super().__init__() - self.react_app_1 = ComponentA() - self.react_app_2 = ComponentB() - - def configure_layout(self): - tab_1 = {"name": "App 1", "content": self.react_app_1} - tab_2 = {"name": "App 2", "content": self.react_app_2} - return tab_1, tab_2 - - - app = L.LightningApp(HelloLitReact()) - -This is a powerful idea that allows each Lightning component to have a self-contained web UI. diff --git a/docs/source-app/workflows/add_web_ui/react/create_react_template.rst b/docs/source-app/workflows/add_web_ui/react/create_react_template.rst deleted file mode 100644 index a5626bc32dcc1..0000000000000 --- a/docs/source-app/workflows/add_web_ui/react/create_react_template.rst +++ /dev/null @@ -1,51 +0,0 @@ -###################################### -Create a React Template (intermediate) -###################################### -**Audience:** Anyone who wants to add a web user interface (UI) written in react to their app. - ----- - -************** -What is react? -************** -`React.js `_ is a JavaScript library for building user interfaces. -A huge number of websites are written in React.js (like Facebook). - ----- - -************************ -Bring your own React app -************************ -If you already have a React.js app, then you don't need the section below. However, it might be helpful -to see our React template so you can understand how to connect it to a Lightning app. - ----- - -**************************** -Create the react-ui template -**************************** -Lightning can generate a react-ui template out of the box (generated with `Vite `_). - -Run this command to set up a react-ui template for a component: - -.. code:: bash - - lightning init react-ui - -If everything was successful, run the example_app.py listed in the output of the command: - -.. code:: bash - - INFO: Checking pre-requisites for react - INFO: - found npm version: 8.5.5 - found node version: 16.15.0 - found yarn version: 1.22.10 - - ... - ... - - ⚡ run the example_app.py to see it live! - lightning run app react-ui/example_app.py - -If the command didn't work, make sure to install `npm+nodejs `_, and `yarn `_. diff --git a/docs/source-app/workflows/add_web_ui/react/index.rst b/docs/source-app/workflows/add_web_ui/react/index.rst deleted file mode 100644 index ba0f8d97d67d1..0000000000000 --- a/docs/source-app/workflows/add_web_ui/react/index.rst +++ /dev/null @@ -1,106 +0,0 @@ -:orphan: - -.. toctree:: - :maxdepth: 1 - :hidden: - - create_react_template - connect_react_and_lightning - communicate_between_react_and_lightning - react_development_workflow - -########################## -Add a web UI with React.js -########################## -**Audience:** Anyone who wants to add a web user interface (UI) written in react to their app. - -**Prereqs:** Basic html knowledge. - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1: Create a React project template - :description: Use our React template to start a react app or bring your own. - :col_css: col-md-6 - :button_link: create_react_template.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: 2: Connect a React app and a Lightning app - :description: Learn how to connect a React app to a Lightning app. - :col_css: col-md-6 - :button_link: connect_react_and_lightning.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: 3: Communicate between React and Lightning - :description: Learn how to communicate between a React app and a Lightning app. - :col_css: col-md-6 - :button_link: communicate_between_react_and_lightning.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: 4: Develop like a React pro - :description: Learn the development workflow of a React developer. - :col_css: col-md-6 - :button_link: react_development_workflow.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
- ----- - -******** -Examples -******** -Here are a few example apps that use a React web UI. - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/react/react_development_workflow.rst b/docs/source-app/workflows/add_web_ui/react/react_development_workflow.rst deleted file mode 100644 index 02d855f383d69..0000000000000 --- a/docs/source-app/workflows/add_web_ui/react/react_development_workflow.rst +++ /dev/null @@ -1,27 +0,0 @@ -######################################### -Add a web UI with React.js (intermediate) -######################################### -**Audience:** Anyone who wants to add a web user interface (UI) written in react to their app. - -**pre-requisites:** You already have a React app connected with a Lightning app. - ----- - -********************** -Develop your react app -********************** -Every time you make a change to your React.js app, you must call `yarn build` to apply the changes (this is a React.js thing): - -.. code:: bash - - # if you're lost, the right folder has a package.json in it - cd folder-with-ui-folder/ui - yarn build - -This can get very repetitive, there is a "hot reload" command that you can enable with: - -.. code:: bash - - # TODO - -There are many other tricks that React.js developers use to improve their development speed. diff --git a/docs/source-app/workflows/add_web_ui/streamlit/basic.rst b/docs/source-app/workflows/add_web_ui/streamlit/basic.rst deleted file mode 100644 index ced0314af54be..0000000000000 --- a/docs/source-app/workflows/add_web_ui/streamlit/basic.rst +++ /dev/null @@ -1,186 +0,0 @@ -################################### -Add a web UI with Streamlit (basic) -################################### -**Audience:** Users who want to add a web UI written with Python. - -**Prereqs:** Basic python knowledge. - ----- - -****************** -What is Streamlit? -****************** -Streamlit is a web user interface builder for Python developers. Streamlit builds beautiful web pages -directly from Python. - -Install Streamlit with: - -.. code:: bash - - pip install streamlit - ----- - -************************* -Run a basic streamlit app -************************* - -.. - To explain how to use Streamlit with Lightning, let's replicate the |st_link|. - - .. |st_link| raw:: html - - example running here - -In the next few sections we'll build an app step-by-step. -First **create a file named app.py** with the app content: - -.. code:: python - - # app.py - import lightning as L - import lightning.app.frontend as frontend - import streamlit as st - - def your_streamlit_app(lightning_app_state): - st.write('hello world') - - class LitStreamlit(L.LightningFlow): - def configure_layout(self): - return frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def run(self): - self.lit_streamlit.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - - app = L.LightningApp(LitApp()) - -add "streamlit" to a requirements.txt file: - -.. code:: bash - - echo 'streamlit' >> requirements.txt - -this is a best practice to make apps reproducible. - ----- - -*********** -Run the app -*********** -Run the app locally to see it! - -.. code:: python - - lightning run app app.py - -Now run it on the cloud as well: - -.. code:: python - - lightning run app app.py --cloud - ----- - -************************ -Step-by-step walkthrough -************************ -In this section, we explain each part of this code in detail. - ----- - -0. Define a streamlit app -^^^^^^^^^^^^^^^^^^^^^^^^^ -First, find the streamlit app you want to integrate. In this example, that app looks like: - -.. code:: python - - import streamlit as st - - def your_streamlit_app(): - st.write('hello world') - -Refer to the `Streamlit documentation `_ for more complex examples. - ----- - -1. Add Streamlit to a component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Link this function to the Lightning App by using the ``StreamlitFrontend`` class which needs to be returned from -the ``configure_layout`` method of the Lightning component you want to connect to Streamlit. - -.. code:: python - :emphasize-lines: 9-11 - - # app.py - import lightning as L - import lightning.app.frontend as frontend - import streamlit as st - - def your_streamlit_app(lightning_app_state): - st.write('hello world') - - class LitStreamlit(L.LightningFlow): - def configure_layout(self): - return frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def run(self): - self.lit_streamlit.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - - app = L.LightningApp(LitApp()) - -The ``render_fn`` argument of the ``StreamlitFrontend`` class, points to a function that runs your Streamlit app. -The first argument to the function is the lightning app state. Any changes to the app state update the app. - ----- - -2. Route the UI in the root component -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The second step, is to tell the Root component in which tab to render this component's UI. -In this case, we render the ``LitStreamlit`` UI in the ``home`` tab of the application. - -.. code:: python - :emphasize-lines: 22 - - # app.py - import lightning as L - import lightning.app.frontend as frontend - import streamlit as st - - def your_streamlit_app(lightning_app_state): - st.write('hello world') - - class LitStreamlit(L.LightningFlow): - def configure_layout(self): - return frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def run(self): - self.lit_streamlit.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - - app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/streamlit/index.rst b/docs/source-app/workflows/add_web_ui/streamlit/index.rst deleted file mode 100644 index 2496729d45660..0000000000000 --- a/docs/source-app/workflows/add_web_ui/streamlit/index.rst +++ /dev/null @@ -1,84 +0,0 @@ -:orphan: - -.. toctree:: - :maxdepth: 1 - :hidden: - - basic - intermediate - -########################### -Add a web UI with Streamlit -########################### - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1: Connect Streamlit - :description: Learn how to connect Streamlit to a Lightning Component. - :col_css: col-md-6 - :button_link: basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: 2: Enable two-way communication - :description: Enable two-way communication between Streamlit and a Lightning App. - :col_css: col-md-6 - :button_link: intermediate.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
- ----- - -******** -Examples -******** -Here are a few example apps that use a Streamlit web UI. - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Example 1 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 2 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. displayitem:: - :header: Example 3 - :description: Show off your work! Contribute an example. - :col_css: col-md-4 - :button_link: ../../../contribute_app.html - :height: 150 - :tag: Waiting for contributed example - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst b/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst deleted file mode 100644 index a89b6e6a5ce55..0000000000000 --- a/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst +++ /dev/null @@ -1,105 +0,0 @@ -########################################## -Add a web UI with Streamlit (intermediate) -########################################## -**Audience:** Users who want to communicate between the Lightning App and Streamlit. - -**Prereqs:** Must have read the :doc:`streamlit basic ` guide. - ----- - -************************************ -Interact with the App from Streamlit -************************************ -The streamlit UI enables user interactions with the Lightning App via UI elements like buttons. -To modify the variables of a Lightning component, access the ``lightning_app_state`` variable in . - -For example, here we increase the count variable of the Lightning Component every time a user presses a button: - -.. code:: python - :emphasize-lines: 8, 14 - - # app.py - import lightning as L - import lightning.app.frontend as frontend - import streamlit as st - - - def your_streamlit_app(lightning_app_state): - if st.button("press to increase count"): - lightning_app_state.count += 1 - st.write(f"current count: {lightning_app_state.count}") - - - class LitStreamlit(L.LightningFlow): - def __init__(self): - super().__init__() - self.count = 0 - - def configure_layout(self): - return frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def run(self): - self.lit_streamlit.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - - - app = L.LightningApp(LitApp()) - ----- - -**************************************** -Interact with Streamlit from a component -**************************************** -To update the streamlit UI from any Lightning component, update the property in the component and make sure to call ``run`` from the -parent component. - -In this example we update the value of the counter from the component: - -.. code:: python - :emphasize-lines: 7, 15 - - # app.py - import lightning as L - import lightning.app.frontend as frontend - import streamlit as st - - - def your_streamlit_app(lightning_app_state): - st.write(f"current count: {lightning_app_state.count}") - - - class LitStreamlit(L.LightningFlow): - def __init__(self): - super().__init__() - self.count = 0 - - def run(self): - self.count += 1 - - def configure_layout(self): - return frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def run(self): - self.lit_streamlit.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - - - app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst b/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst deleted file mode 100644 index e8d9f3e843155..0000000000000 --- a/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst +++ /dev/null @@ -1,6 +0,0 @@ -:orphan: - -####################################### -Add a web UI with Vue.js (intermediate) -####################################### -coming... diff --git a/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst b/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst deleted file mode 100644 index 91c0e53854760..0000000000000 --- a/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst +++ /dev/null @@ -1,69 +0,0 @@ -######################## -Arrange app tabs (basic) -######################## -**Audience:** Users who want to control the layout of their app user interface. - ----- - -***************************** -Enable a full-page single tab -***************************** - -To enable a single tab on the app UI, return a single dictionary from the ``configure_layout`` method: - -.. code:: python - :emphasize-lines: 9 - - import lightning as L - - - class DemoComponent(L.demo.dumb_component): - def configure_layout(self): - tab1 = {"name": "THE TAB NAME", "content": self.component_a} - return tab1 - - - app = L.LightningApp(DemoComponent()) - - -The "name" key defines the visible name of the tab on the UI. It also shows up in the URL. -The **"content"** key defines the target component to render in that tab. -When returning a single tab element like shown above, the UI will display it in full-page mode. - - ----- - -******************** -Enable multiple tabs -******************** - -.. code:: python - :emphasize-lines: 7 - - import lightning as L - - - class DemoComponent(L.demo.dumb_component): - def configure_layout(self): - tab1 = {"name": "Tab A", "content": self.component_a} - tab2 = {"name": "Tab B", "content": self.component_b} - return tab1, tab2 - - - app = L.LightningApp(DemoComponent()) - -The order matters! Try any of the following configurations: - -.. code:: python - :emphasize-lines: 4, 9 - - def configure_layout(self): - tab1 = {"name": "Tab A", "content": self.component_a} - tab2 = {"name": "Tab B", "content": self.component_b} - return tab1, tab2 - - - def configure_layout(self): - tab1 = {"name": "Tab A", "content": self.component_a} - tab2 = {"name": "Tab B", "content": self.component_b} - return tab2, tab1 diff --git a/docs/source-app/workflows/arrange_tabs/arrange_app_intermediate.rst b/docs/source-app/workflows/arrange_tabs/arrange_app_intermediate.rst deleted file mode 100644 index 1bb638b1d2c0c..0000000000000 --- a/docs/source-app/workflows/arrange_tabs/arrange_app_intermediate.rst +++ /dev/null @@ -1,21 +0,0 @@ -############################### -Arrange app tabs (intermediate) -############################### - -.. TODO:: fill-in - ----- - -*********************************** -Render components with a defined UI -*********************************** - -component directly - ----- - -************* -Render a link -************* - -tensorboard link diff --git a/docs/source-app/workflows/arrange_tabs/index.rst b/docs/source-app/workflows/arrange_tabs/index.rst deleted file mode 100644 index f639c0238e1bb..0000000000000 --- a/docs/source-app/workflows/arrange_tabs/index.rst +++ /dev/null @@ -1,5 +0,0 @@ -################ -Arrange App Tabs -################ - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/arrange_tabs/index_content.rst b/docs/source-app/workflows/arrange_tabs/index_content.rst deleted file mode 100644 index 66ac9328e67a7..0000000000000 --- a/docs/source-app/workflows/arrange_tabs/index_content.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :hidden: - - arrange_app_basic - arrange_app_intermediate - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Basic - :description: Learn how to enable and layout your app UI - :col_css: col-md-6 - :button_link: arrange_app_basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Intermediate - :description: Learn about all the possible ways of rendering a component. - :col_css: col-md-6 - :button_link: arrange_app_intermediate.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_command_line_interface/app.py b/docs/source-app/workflows/build_command_line_interface/app.py deleted file mode 100644 index 373aa2bfd137e..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/app.py +++ /dev/null @@ -1,35 +0,0 @@ -from commands.notebook.run import RunNotebook, RunNotebookConfig -from lit_jupyter import JupyterLab - -from lightning.app import LightningFlow, LightningApp, CloudCompute -from lightning.app.structures import Dict - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.notebooks = Dict() - - # 1. Annotates the handler input with the Notebook config. - def run_notebook(self, config: RunNotebookConfig): - if config.name in self.notebooks: - return f"The Notebook {config.name} already exists." - else: - # 2. Dynamically creates the Notebook if it doesn't exist and runs it. - self.notebooks[config.name] = JupyterLab( - cloud_compute=CloudCompute(config.cloud_compute) - ) - self.notebooks[config.name].run() - return f"The Notebook {config.name} was created." - - def configure_commands(self): - # 3. Returns a list of dictionaries with the format: - # {"command_name": CustomClientCommand(method=self.custom_server_handler)} - return [{"run notebook": RunNotebook(method=self.run_notebook)}] - - def configure_layout(self): - # 4. Dynamically displays the Notebooks in the Lightning App View. - return [{"name": n, "content": w} for n, w in self.notebooks.items()] - - -app = LightningApp(Flow()) diff --git a/docs/source-app/workflows/build_command_line_interface/cli.rst b/docs/source-app/workflows/build_command_line_interface/cli.rst deleted file mode 100644 index 176e416b31630..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/cli.rst +++ /dev/null @@ -1,144 +0,0 @@ -:orphan: - -########################################### -1. Develop a CLI with server side code only -########################################### - -We are going to learn how to create a simple command-line interface. - -Lightning provides a flexible way to create complex CLI without much effort. - ----- - -************************* -1. Implement a simple CLI -************************* - -To create your first CLI, you need to override the :class:`~lightning.app.core.flow.LightningFlow.configure_commands` hook and return a list of dictionaries where the keys are the commands and the values are the server side handlers. - -First, create a file ``app.py`` and copy-paste the following code in to the file: - -.. literalinclude:: example_command.py - ----- - -************** -2. Run the App -************** - -Execute the following command in a terminal: - -.. code-block:: - - lightning_app run app app.py - -The following appears the terminal: - -.. code-block:: - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - [] - ----- - -*************************** -3. Connect to a running App -*************************** - -In another terminal, connect to the running App. -When you connect to an App, the Lightning CLI is replaced by the App CLI. To exit the App CLI, you need to run ``lightning_app disconnect``. - -.. code-block:: - - lightning_app connect localhost - -To see a list of available commands: - -.. code-block:: - - lightning_app --help - You are connected to the cloud Lightning App: localhost. - Usage: lightning_app [OPTIONS] COMMAND [ARGS]... - - --help Show this message and exit. - - Lightning App Commands - add Add a name. - -To find the arguments of the commands: - -.. code-block:: - - lightning_app add --help - You are connected to the cloud Lightning App: localhost. - Usage: lightning_app add [ARGS]... - - Options - name: Add description - ----- - -******************** -4. Execute a command -******************** - -Trigger the command line exposed by your App: - -.. code-block:: - - lightning_app add --name=my_name - WARNING: Lightning Command Line Interface is an experimental feature and unannounced changes are likely. - -In your first terminal, **Received name: my_name** and **["my_name"]** are printed. - -.. code-block:: - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - [] - Received name: my_name - ["my_name] - ----- - -************************** -5. Disconnect from the App -************************** - -To exit the App CLI, you need to run ``lightning_app disconnect``. - -.. code-block:: - - lightning_app disconnect - You are disconnected from the local Lightning App. - ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 2. Implement a CLI with client side code execution - :description: Learn how to develop a complex API for your application - :col_css: col-md-6 - :button_link: cli_client.html - :height: 150 - -.. displayitem:: - :header: Develop a RESTful API - :description: Learn how to develop an API for your application. - :col_css: col-md-6 - :button_link: ../build_rest_api/index.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_command_line_interface/cli_client.rst b/docs/source-app/workflows/build_command_line_interface/cli_client.rst deleted file mode 100644 index 589db930bf5fa..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/cli_client.rst +++ /dev/null @@ -1,175 +0,0 @@ -:orphan: - -###################################################### -2. Develop a CLI with server and client code execution -###################################################### - -We've learned how to create a simple command-line interface. But in real-world use-cases, an App Builder wants to provide more complex functionalities where trusted code is executed on the client side. - -Lightning provides a flexible way to create complex CLI without much effort. - -In this example, we’ll create a CLI to dynamically run Notebooks: - - ----- - -************************** -1. Implement a complex CLI -************************** - -First of all, lets' create the following file structure: - -.. code-block:: python - - app_folder/ - commands/ - notebook/ - run.py - app.py - -We'll use the `Jupyter-Component `_. Follow the installation steps on the repo to install the Component. - -Add the following code to ``commands/notebook/run.py``: - -.. literalinclude:: commands/notebook/run.py - -Add the following code to ``app.py``: - -.. literalinclude:: app.py - ----- - -********************************************** -2. Run the App and check the API documentation -********************************************** - -In a terminal, run the following command and open ``http://127.0.0.1:7501/docs`` in a browser. - -.. code-block:: python - - lightning_app run app app.py - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - ----- - -*************************** -3. Connect to a running App -*************************** - -In another terminal, connect to the running App. -When you connect to an App, the Lightning CLI is replaced by the App CLI. To exit the App CLI, you need to run ``lightning_app disconnect``. - -.. code-block:: - - lightning_app connect localhost - - Storing `run_notebook` under /Users/thomas/.lightning/lightning_connection/commands/run_notebook.py - You can review all the downloaded commands under /Users/thomas/.lightning/lightning_connection/commands folder. - You are connected to the local Lightning App. - -To see a list of available commands: - -.. code-block:: - - lightning_app --help - - You are connected to the cloud Lightning App: localhost. - Usage: lightning_app [OPTIONS] COMMAND [ARGS]... - - --help Show this message and exit. - - Lightning App Commands - run notebook Run a Notebook. - - -To find the arguments of the commands: - -.. code-block:: - - lightning_app run notebook --help - - You are connected to the cloud Lightning App: localhost. - usage: notebook [-h] [--name NAME] [--cloud_compute CLOUD_COMPUTE] - - Run Notebook Parser - - optional arguments: - -h, --help show this help message and exit - --name NAME - --cloud_compute CLOUD_COMPUTE - ----- - -******************** -4. Execute a command -******************** - -And then you can trigger the command-line exposed by your App. - -Run the first Notebook with the following command: - -.. code-block:: python - - lightning_app run notebook --name="my_notebook" - WARNING: Lightning Command Line Interface is an experimental feature and unannounced changes are likely. - The notebook my_notebook was created. - -And run a second notebook. - -.. code-block:: python - - lightning_app run notebook --name="my_notebook_2" - WARNING: Lightning Command Line Interface is an experimental feature and unannounced changes are likely. - The notebook my_notebook_2 was created. - -Here is a recording of the Lightning App: - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/commands_1.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/commands_1.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -************************** -5. Disconnect from the App -************************** - -To exit the App CLI, you need to run **lightning disconnect**. - -.. code-block:: - - lightning_app disconnect - You are disconnected from the local Lightning App. - ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Develop a CLI with server side code only - :description: Learn how to develop a simple CLI for your App. - :col_css: col-md-6 - :button_link: cli.html - :height: 150 - -.. displayitem:: - :header: Develop a RESTful API - :description: Learn how to develop an API for your App. - :col_css: col-md-6 - :button_link: ../build_rest_api/index.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_command_line_interface/commands/__init__.py b/docs/source-app/workflows/build_command_line_interface/commands/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/source-app/workflows/build_command_line_interface/commands/notebook/__init__.py b/docs/source-app/workflows/build_command_line_interface/commands/notebook/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/source-app/workflows/build_command_line_interface/commands/notebook/run.py b/docs/source-app/workflows/build_command_line_interface/commands/notebook/run.py deleted file mode 100644 index e0a6463c2d6a3..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/commands/notebook/run.py +++ /dev/null @@ -1,33 +0,0 @@ -from argparse import ArgumentParser -from uuid import uuid4 - -from pydantic import BaseModel - -from lightning.app.utilities.commands import ClientCommand - - -class RunNotebookConfig(BaseModel): - name: str - cloud_compute: str - - -class RunNotebook(ClientCommand): - description = "Run a Notebook." - - def run(self): - # 1. Define your own argument parser. You can use argparse, click, etc... - parser = ArgumentParser(description='Run Notebook Parser') - parser.add_argument("--name", type=str, default=None) - parser.add_argument("--cloud_compute", type=str, default="cpu") - hparams = parser.parse_args() - - # 2. Invoke the server side handler by sending a payload. - response = self.invoke_handler( - config=RunNotebookConfig( - name=hparams.name or str(uuid4()), - cloud_compute=hparams.cloud_compute, - ), - ) - - # 3. Print the server response. - print(response) diff --git a/docs/source-app/workflows/build_command_line_interface/example_command.py b/docs/source-app/workflows/build_command_line_interface/example_command.py deleted file mode 100644 index 4d837fc007171..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/example_command.py +++ /dev/null @@ -1,25 +0,0 @@ -from lightning import LightningApp, LightningFlow - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.names = [] - - def run(self): - print(self.names) - - def add_name(self, name: str): - """Add a name.""" - print(f"Received name: {name}") - self.names.append(name) - - def configure_commands(self): - # This can be invoked with `lightning add --name=my_name` - commands = [ - {"add": self.add_name}, - ] - return commands - - -app = LightningApp(Flow()) diff --git a/docs/source-app/workflows/build_command_line_interface/index.rst b/docs/source-app/workflows/build_command_line_interface/index.rst deleted file mode 100644 index 9a3e24f784910..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/index.rst +++ /dev/null @@ -1,48 +0,0 @@ -############################ -Command-line Interface (CLI) -############################ - -**Audience:** Users looking to create a command line interface (CLI) for their application. - ----- - -************** -What is a CLI? -************** - -A Command-line Interface (CLI) is an user interface (UI) in a terminal to interact with a specific program. - -.. note:: - - The Lightning guideline to build CLI is `lightning_app ...` or ` ...`. - -As an example, Lightning provides a CLI to interact with your Lightning Apps and the `lightning.ai `_ platform as follows: - -.. code-block:: bash - - main - ├── fork - Forks an App. - ├── init - Initializes a Lightning App and/or Component. - │ ├── app - │ ├── component - │ ├── pl-app - Creates an App from your PyTorch Lightning source files. - │ └── react-ui - Creates a React UI to give a Lightning Component a React.js web UI - ├── install - Installs a Lightning App and/or Component. - │ ├── app - │ └── component - ├── list - Lists Lightning AI self-managed resources (apps) - │ └── apps - Lists your Lightning AI Apps. - ├── login - Logs in to your lightning.ai account. - ├── logout - Logs out of your lightning.ai account. - ├── run - Runs a Lightning App locally or on the cloud. - │ └── app - Runs an App from a file. - ├── show - Shows given resource. - │ └── logs - Shows cloud application logs. By default prints logs for all currently available Components. - ├── stop - Stops your App. - └── tree - Shows the command tree of your CLI. - -Learn more about `Command-line interfaces here `_. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/build_command_line_interface/index_content.rst b/docs/source-app/workflows/build_command_line_interface/index_content.rst deleted file mode 100644 index ced369dbfd815..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/index_content.rst +++ /dev/null @@ -1,51 +0,0 @@ -************************************** -Develop a command line interface (CLI) -************************************** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Develop a CLI with server side code only - :description: Learn how to develop a simple CLI for your application - :col_css: col-md-6 - :button_link: cli.html - :height: 150 - -.. displayitem:: - :header: 2. Develop a CLI with server and client code execution - :description: Learn how to develop a complex CLI for your application - :col_css: col-md-6 - :button_link: cli_client.html - :height: 150 - -.. raw:: html - -
-
- - ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a RESTful API - :description: Learn how to develop an API for your application. - :col_css: col-md-6 - :button_link: ../build_rest_api/index.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_command_line_interface/post_example.py b/docs/source-app/workflows/build_command_line_interface/post_example.py deleted file mode 100644 index c57a2d9426539..0000000000000 --- a/docs/source-app/workflows/build_command_line_interface/post_example.py +++ /dev/null @@ -1,26 +0,0 @@ -from lightning.app import LightningFlow, LightningApp -from lightning.app.api import Post - - -class Flow(LightningFlow): - # 1. Define the state - def __init__(self): - super().__init__() - self.names = [] - - # 2. Optional, but used to validate names - def run(self): - print(self.names) - - # 3. Method executed when a request is received. - def handle_post(self, name: str): - self.names.append(name) - return f'The name {name} was registered' - - # 4. Defines this Component's Restful API. You can have several routes. - def configure_api(self): - # Your own defined route and handler - return [Post(route="/name", method=self.handle_post)] - - -app = LightningApp(Flow()) diff --git a/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst b/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst deleted file mode 100644 index f489edaedc13f..0000000000000 --- a/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst +++ /dev/null @@ -1,109 +0,0 @@ -####################################################### -Develop a Lightning App from a PyTorch Lightning script -####################################################### - -**Audience:** Users who want to develop a Lightning App (App) from their PyTorch Lightning (PL) scripts. - ----- - -************************************************************* -What developing a Lightning App from a PL script does for you -************************************************************* - -Developing an App from a PL script allows you to immediately run on the cloud and share the progress with friends. -Once you're happy with your model, you can immediately expand beyond just model development to things like -making your own inference APIs, research demos, or even speeding up your data pipeline. - -The PyTorch Lightning App is your entry point to the full end-to-end ML licefycle. - ----- - -****************** -Develop a template -****************** - -To develop a template from a PyTorch Lightning script, use this command: - -.. code:: bash - - lightning_app init pl-app path/to/the/pl_script.py - - -If your script is not at the root of the project folder, and you'd like to include all source files within that folder, you can specify the root path as the first argument: - -.. code:: bash - - lightning_app init pl-app path/to/project/root path/to/the/pl_script.py - - -The default trainer App lets you train a model with a beautiful UI locally and on the cloud with zero effort! - ----- - -*********** -Run the App -*********** - -.. note:: This section is under construction. - -Run the App locally: - -.. code:: bash - - lightning_app run app pl-app/app.py - -Or run the App on the cloud so you can share with collaborators and even use all the cloud GPUs you want. - -.. code:: bash - - lightning_app run app pl-app/app.py --cloud - - -.. figure:: https://storage.googleapis.com/grid-packages/pytorch-lightning-app/docs-thumbnail.png - :alt: Screenshot of the PyTorch Lightning app running in the cloud - - ----- - -******************* -Modify the template -******************* - -The command above generates an App file like this: - -.. TODO:: list the file and show how to extend it - -.. code:: python - - from your_app_name import ComponentA, ComponentB - - import lightning as L - - - class LitApp(L.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component_a = ComponentA() - self.component_b = ComponentB() - - def run(self): - self.component_a.run() - self.component_b.run() - - - app = L.LightningApp(LitApp()) - -Now you can add your own components as you wish! - ----- - -************ -Known issues -************ - -- The UI takes a couple seconds to load when opening the App, so please be patient. -- The timer resets when refreshing the page. -- The UI for adding new environment variables does not provide an option to delete an entry. -- A bug exists that leaves the script hanging at the start of training when using the DDP strategy. -- DDP-spawn is not supported due to pickling issues. -- It is currently not possible to submit a new run once the script has finished or failed. diff --git a/docs/source-app/workflows/build_lightning_app/from_scratch.rst b/docs/source-app/workflows/build_lightning_app/from_scratch.rst deleted file mode 100644 index 9042f105711cd..0000000000000 --- a/docs/source-app/workflows/build_lightning_app/from_scratch.rst +++ /dev/null @@ -1,11 +0,0 @@ -#################################### -Develop a Lightning App from Scratch -#################################### - -**Audience:** Users who want to develop a Lightning App from scratch. - -**Prereqs:** You must have finished the `Basic levels `_. - ----- - -.. include:: from_scratch_content.rst diff --git a/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst b/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst deleted file mode 100644 index a528fa6088736..0000000000000 --- a/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst +++ /dev/null @@ -1,60 +0,0 @@ - -************** -WAIT! -************** -Before you build a Lightning App from scratch, see if you can find an app that is similar to what you need -in the `Lightning App Gallery `_. - -Once you find the Lightning App you want, press "Clone & Run" to see it running on the cloud, then download the code -and change what you want! - ----- - -****************** -Build from scratch -****************** -If you didn't find a Lightning App similar to the one you need, simply create a file named **app.py** with these contents: - -.. code:: python - - import lightning as L - - - class WordComponent(L.LightningWork): - def __init__(self, word): - super().__init__() - self.word = word - - def run(self): - print(self.word) - - - class LitApp(L.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.hello = WordComponent("hello") - self.world = WordComponent("world") - - def run(self): - print("This is a simple Lightning app, make a better app!") - self.hello.run() - self.world.run() - - - app = L.LightningApp(LitApp()) - ----- - -Run the Lightning App -^^^^^^^^^^^^^^^^^^^^^ -Run the Lightning App locally: - -.. code:: bash - - lightning_app run app app.py - -Run the Lightning App on the cloud: - -.. code:: bash - - lightning_app run app app.py --cloud diff --git a/docs/source-app/workflows/build_lightning_app/index.rst b/docs/source-app/workflows/build_lightning_app/index.rst deleted file mode 100644 index e60f0355afb8e..0000000000000 --- a/docs/source-app/workflows/build_lightning_app/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -####################### -Develop a Lightning App -####################### - -A Lightning App (App) is a collection of components interacting together. Learn how to develop a basic App template. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/build_lightning_app/index_content.rst b/docs/source-app/workflows/build_lightning_app/index_content.rst deleted file mode 100644 index 45264d85a4adc..0000000000000 --- a/docs/source-app/workflows/build_lightning_app/index_content.rst +++ /dev/null @@ -1,32 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :hidden: - - from_scratch - from_pytorch_lightning_script - -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a Lightning App from scratch - :description: Learn how to Develop a Lightning App from scratch - :col_css: col-md-6 - :button_link: from_scratch.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Develop an App from a PyTorch Lightning script - :description: Share your PyTorch Lightning training on the cloud, run on cloud GPUs, or extend your App - :col_css: col-md-6 - :button_link: from_pytorch_lightning_script.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_lightning_component/basic.rst b/docs/source-app/workflows/build_lightning_component/basic.rst deleted file mode 100644 index 07fac58cf21da..0000000000000 --- a/docs/source-app/workflows/build_lightning_component/basic.rst +++ /dev/null @@ -1,9 +0,0 @@ -############################# -Develop a Lightning Component -############################# - -**Audience:** Users who want to develop a Lightning Component. - ----- - -.. include:: from_scratch_component_content.rst diff --git a/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst b/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst deleted file mode 100644 index b29c19db40129..0000000000000 --- a/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst +++ /dev/null @@ -1,153 +0,0 @@ -******************************* -LightningFlow vs. LightningWork -******************************* - -.. _flow_vs_work: - -.. raw:: html - - Choosing between LightningFlow and LightningWork - -There are two types of components in Lightning, **LightningFlow** and **LightningWork**. - -Use a **LightningFlow** component for any programming logic that runs in less than 1 second. - -.. code:: python - - for i in range(10): - print(f"{i}: this kind of code belongs in a LightningFlow") - -Use a **LightningWork** component for any programming logic that takes more than 1 second or requires its own hardware. - -.. code:: python - - from time import sleep - - for i in range(100000): - sleep(2.0) - print(f"{i} LightningWork: work that is long running or may never end (like a server)") - ----- - -************************************************** -What developing a Lightning Component does for you -************************************************** -Lightning Components break up complex systems into modular components. The first obvious benefit is that components -can be reused across other apps. This means you can build once, test it and forget it. - -As a researcher it also means that your code can be taken to production without needing a team of engineers to help -productionize it. - -As a machine learning engineer, it means that your cloud system is: - -- fault tolerant -- cloud agnostic -- testable (unlike YAML/CI/CD code) -- version controlled -- enables cross-functional collaboration - ----- - -************** -WAIT! -************** -Before you build a Lightning component from scratch, see if you can find a component that is similar to what you need -in the `Lightning component Gallery `_. - -Once you find the component you want, download the code and change what you want! - ----- - -***************************************** -Build a Lighitning component from scratch -***************************************** -If you didn't find a Lightning component similar to the one you need, you can build one from scratch. - ----- - -Build a LightningFlow -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To implement a LightningFlow, simply subclass ``LightningFlow`` and define the run method: - -.. code:: python - :emphasize-lines: 5 - - # app.py - import lightning as L - - - class LitFlow(L.LightningFlow): - def run(self): - for i in range(10): - print(f"{i}: this kind of code belongs in a LightningFlow") - - - app = L.LightningApp(LitFlow()) - -run the app - -.. code:: bash - - lightning_app run app app.py - ----- - -Build a LightningWork -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Only implement a LightningWork if this particular piece of code: - -- takes more than 1 second to execute -- requires its own set of cloud resources -- or both - -To implement a LightningWork, simply subclass ``LightningWork`` and define the run method: - -.. code:: python - :emphasize-lines: 6 - - # app.py - from time import sleep - import lightning as L - - - class LitWork(L.LightningWork): - def run(self): - for i in range(100000): - sleep(2.0) - print(f"{i} LightningWork: work that is long running or may never end (like a server)") - -A LightningWork must always be attached to a LightningFlow and explicitly asked to ``run()``: - -.. code:: python - :emphasize-lines: 13, 16 - - from time import sleep - import lightning as L - - - class LitWork(L.LightningWork): - def run(self): - for i in range(100000): - sleep(2.0) - print(f"{i} LightningWork: work that is long running or may never end (like a server)") - - - class LitFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_work = LitWork() - - def run(self): - self.lit_work.run() - - - app = L.LightningApp(LitFlow()) - -run the app - -.. code:: bash - - lightning_app run app app.py diff --git a/docs/source-app/workflows/build_lightning_component/index.rst b/docs/source-app/workflows/build_lightning_component/index.rst deleted file mode 100644 index 8620a9b9fd8d5..0000000000000 --- a/docs/source-app/workflows/build_lightning_component/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -############################# -Develop a Lightning Component -############################# - -A Lightning App (App) is a collection of components interacting together. Learn how to build a Lightning Component (Component) in this section. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/build_lightning_component/index_content.rst b/docs/source-app/workflows/build_lightning_component/index_content.rst deleted file mode 100644 index 9a940bc8b89b6..0000000000000 --- a/docs/source-app/workflows/build_lightning_component/index_content.rst +++ /dev/null @@ -1,122 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :hidden: - - basic - ../add_components - -.. toctree:: - :maxdepth: 1 - :hidden: - - intermediate - ../run_work_in_parallel - ../run_work_once - -.. toctree:: - :maxdepth: 1 - :hidden: - - ../enable_fault_tolerance - -****** -Basics -****** -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a Lightning Component - :description: Learn the basics of developing a Lightning Component - :col_css: col-md-4 - :button_link: basic.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Explore community Lightning Components - :description: Discover community-built Lightning Components - :col_css: col-md-4 - :button_link: https://lightning.ai/components - :height: 150 - :tag: basic - -.. raw:: html - -
-
- ----- - -************ -Intermediate -************ - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add a UI to a component - :description: Learn about all the possible ways of rendering a component. - :col_css: col-md-4 - :button_link: intermediate.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Run LightningWork in parallel - :description: Learn about running LightningWork in parallel. - :col_css: col-md-4 - :button_link: ../run_work_in_parallel.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Run LightningWork once - :description: Learn about running LightningWork multiple times or once. - :col_css: col-md-4 - :button_link: ../run_work_once.html - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Publish a Lightning component - :description: Learn the basics of publishing a Lightning component. - :col_css: col-md-4 - :button_link: publish_a_component.html - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
- - ----- - -******** -Advanced -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Enable fault tolerance - :description: Learn how to make a component fault tolerant. - :col_css: col-md-4 - :button_link: ../enable_fault_tolerance.html - :height: 150 - :tag: advanced - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_lightning_component/intermediate.rst b/docs/source-app/workflows/build_lightning_component/intermediate.rst deleted file mode 100644 index b95ba36ca4c2d..0000000000000 --- a/docs/source-app/workflows/build_lightning_component/intermediate.rst +++ /dev/null @@ -1,71 +0,0 @@ -############################################ -Develop a Lightning Component (intermediate) -############################################ - -**Audience:** Users who want to connect a UI to a Lightning Component (Component). - ----- - -***************************** -Add a web user interface (UI) -***************************** -Every lightning component can have its own user interface (UI). Lightning components support any kind -of UI interface such as dash, gradio, panel, react.js, streamlit, vue.js, web urls, -etc...(:doc:`full list here <../add_web_ui/index>`). - -Let's say that we have a user interface defined in html: - -.. code:: html - - - - - - -

Hello World

- - - -To *connect* this user interface to the Component, define the configure_layout method: - -.. code:: python - :emphasize-lines: 5, 6 - - import lightning as L - - - class LitHTMLComponent(L.LightningFlow): - def configure_layout(self): - return L.app.frontend.StaticWebFrontend(serve_dir="path/to/folder/with/index.html/inside") - -Finally, route the Component's UI through the root Component's **configure_layout** method: - -.. code:: python - :emphasize-lines: 14 - - # app.py - import lightning as L - - - class LitHTMLComponent(L.LightningFlow): - def configure_layout(self): - return L.app.frontend.StaticWebFrontend(serve_dir="path/to/folder/with/index.html/inside") - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_html_component = LitHTMLComponent() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_html_component} - return tab1 - - - app = L.LightningApp(LitApp()) - -Run your App and you'll see the UI on the Lightning App view: - -.. code:: bash - - lightning run app app.py diff --git a/docs/source-app/workflows/build_lightning_component/publish_a_component.rst b/docs/source-app/workflows/build_lightning_component/publish_a_component.rst deleted file mode 100644 index bb5ec755ae190..0000000000000 --- a/docs/source-app/workflows/build_lightning_component/publish_a_component.rst +++ /dev/null @@ -1,59 +0,0 @@ -############################# -Publish a Lightning Component -############################# - -**Audience:** Users who want to build a Ligthtning Component (Component) to publish to the Lightning Gallery - ----- - -*********************************** -Develop a Component from a template -*********************************** - -The fastest way to build a Component that is ready to be published to the component Gallery is to use -the default template. - -Generate your Component template with this command: - -.. code:: python - - lightning init component your-component-name - ----- - -***************** -Run the Component -***************** - -To test that your Component works, first install all dependencies: - -.. code:: bash - - cd your-component - pip install -r requirements.txt - pip install -e . - -Now import your Component and use it in a Lightning App: - -.. code:: python - - # app.py - from your_component import TemplateComponent - import lightning as L - - class LitApp(L.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.your_component = TemplateComponent() - - def run(self): - print('this is a simple Lightning app to verify your component is working as expected') - self.your_component.run() - - app = L.LightningApp(LitApp()) - -and run the app: - -.. code:: bash - - lightning run app app.py diff --git a/docs/source-app/workflows/build_rest_api/add_api.rst b/docs/source-app/workflows/build_rest_api/add_api.rst deleted file mode 100644 index 00fd16d18715a..0000000000000 --- a/docs/source-app/workflows/build_rest_api/add_api.rst +++ /dev/null @@ -1,104 +0,0 @@ -:orphan: - -############################ -Add an API Route to your App -############################ - -In order to add a new route, you need to override the :class:`~lightning.app.core.flow.LightningFlow.configure_api` hook and return a list of :class:`~lightning.app.api.http_methods.HttpMethod` such as :class:`~lightning.app.api.http_methods.Get`, :class:`~lightning.app.api.http_methods.Post`, :class:`~lightning.app.api.http_methods.Put`, :class:`~lightning.app.api.http_methods.Delete`. - ----- - -********************** -1. Create a simple App -********************** - -We're going to create a single route ``/name`` that takes a string input ``name`` and stores the value within the ``names`` attribute of the flow state. - -Create a file called ``app.py`` and copy-paste the following code in to the file: - -.. literalinclude:: post_example.py - ----- - -************** -2. Run the App -************** - -Execute the following command in a terminal: - -.. code-block:: - - lightning_app run app app.py - -The following appears: - -.. code-block:: - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - ----- - -**************** -3. Check the API -**************** - -The Lightning App framework automatically generates API documentation from your App using `Swagger UI `_. - -You can access it by accessing the following URL: ``http://127.0.0.1:7501/docs`` in your browser and validate your API with the route ``/name`` directly from the documentation page as shown below. - -.. video:: https://pl-public-data.s3.amazonaws.com/assets_lightning/rest_post.mp4 - :poster: https://pl-public-data.s3.amazonaws.com/assets_lightning/rest_png.png - :width: 600 - :class: background-video - :autoplay: - :loop: - :muted: - -Alternatively, you can invoke the route directly from a second terminal using `curl `_. - -.. code-block:: - - curl -X 'POST' \ - 'http://127.0.0.1:7501/name?name=my_name' \ - -H 'accept: application/json' \ - -d '' - - "The name my_name was registered" - -And you can see the following in your first terminal running your App. - -.. code-block:: - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - [] - ["my_name"] - -************************************** -Develop a command line interface (CLI) -************************************** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add Requests Validation - :description: Learn how to use pydantic with your API. - :col_css: col-md-6 - :button_link: request_validation.html - :height: 150 - -.. displayitem:: - :header: Develop a Command Line Interface (CLI) - :description: Learn how to develop an CLI for your App. - :col_css: col-md-6 - :button_link: ../build_command_line_interface/index.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_rest_api/index.rst b/docs/source-app/workflows/build_rest_api/index.rst deleted file mode 100644 index 590f3d03d538d..0000000000000 --- a/docs/source-app/workflows/build_rest_api/index.rst +++ /dev/null @@ -1,34 +0,0 @@ -:orphan: - -########### -RESTful API -########### - -**Audience:** Users looking to create an API in their App to allow users to activate functionalities from external sources. - ----- - -********************** -What is a RESTful API? -********************** - -A RESTful API is a set of external URL routes exposed by a server that enables clients to trigger some functionalities, such as getting or putting some data, uploading files, etc.. - -This provides great flexibility for users as they can easily discover functionalities made available by the App Builders. - -The Lightning App framework supports the four primary HTTP methods: `GET`, `POST`, `PUT`, `DELETE`. - -These methods are guidelines to organize your RESTful Services and help users understand your functionalities. - -* **`GET`:** Reads data from the server. -* **`POST`:** Creates new resources. -* **`PUT`:** Updates/replaces existing resources. -* **`DELETE`:** Deletes resources. - -Learn more about `HTTP Methods for RESTful Services here `_. - -The Lightning App framework uses the popular `FastAPI `_ and `Pydantic `_ frameworks under the hood. This means you can use all their features while building your App. - ----- - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/build_rest_api/index_content.rst b/docs/source-app/workflows/build_rest_api/index_content.rst deleted file mode 100644 index 9f77225f24f59..0000000000000 --- a/docs/source-app/workflows/build_rest_api/index_content.rst +++ /dev/null @@ -1,50 +0,0 @@ -************** -Develop an API -************** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add an API Route to your App - :description: Learn how to develop a simple API for your App. - :col_css: col-md-6 - :button_link: add_api.html - :height: 150 - -.. displayitem:: - :header: Add Requests Validation - :description: Learn how to use pydantic with your API. - :col_css: col-md-6 - :button_link: cli_client.html - :height: 150 - -.. raw:: html - -
-
- ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Develop a Command-line Interface - :description: Learn how to develop an CLI for your App. - :col_css: col-md-6 - :button_link: ../build_command_line_interface/index.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/build_rest_api/models.py b/docs/source-app/workflows/build_rest_api/models.py deleted file mode 100644 index 7ebb3ac8c8c17..0000000000000 --- a/docs/source-app/workflows/build_rest_api/models.py +++ /dev/null @@ -1,6 +0,0 @@ -from pydantic import BaseModel - - -# 1. Subclass the BaseModel and defines your payload format. -class NamePostConfig(BaseModel): - name: str diff --git a/docs/source-app/workflows/build_rest_api/post_example.py b/docs/source-app/workflows/build_rest_api/post_example.py deleted file mode 100644 index 0e56117a44fcf..0000000000000 --- a/docs/source-app/workflows/build_rest_api/post_example.py +++ /dev/null @@ -1,25 +0,0 @@ -from lightning.app import LightningFlow, LightningApp -from lightning.app.api import Post - - -class Flow(LightningFlow): - # 1. Define the state - def __init__(self): - super().__init__() - self.names = [] - - # 2. Optional, but used to validate names - def run(self): - print(self.names) - - # 3. Method executed when a request is received. - def handle_post(self, name: str): - self.names.append(name) - return f'The name {name} was registered' - - # 4. Defines this Component's Restful API. You can have several routes. - def configure_api(self): - return [Post(route="/name", method=self.handle_post)] - - -app = LightningApp(Flow()) diff --git a/docs/source-app/workflows/build_rest_api/post_example_pydantic.py b/docs/source-app/workflows/build_rest_api/post_example_pydantic.py deleted file mode 100644 index 35ae78186f538..0000000000000 --- a/docs/source-app/workflows/build_rest_api/post_example_pydantic.py +++ /dev/null @@ -1,32 +0,0 @@ -from models import NamePostConfig # 2. Import your custom model. - -from lightning.app import LightningFlow, LightningApp -from lightning.app.api import Post - - -class Flow(LightningFlow): - # 1. Define the state - def __init__(self): - super().__init__() - self.names = [] - - # 2. Optional, but used to validate names - def run(self): - print(self.names) - - # 3. Annotate your input with your custom pydantic model. - def handle_post(self, config: NamePostConfig): - self.names.append(config.name) - return f'The name {config} was registered' - - # 4. Defines this Component's Restful API. You can have several routes. - def configure_api(self): - return [ - Post( - route="/name", - method=self.handle_post, - ) - ] - - -app = LightningApp(Flow()) diff --git a/docs/source-app/workflows/build_rest_api/request_validation.rst b/docs/source-app/workflows/build_rest_api/request_validation.rst deleted file mode 100644 index 6caaccdc239d2..0000000000000 --- a/docs/source-app/workflows/build_rest_api/request_validation.rst +++ /dev/null @@ -1,69 +0,0 @@ -:orphan: - -*********************** -Add Requests Validation -*********************** - -The Lightning App framework uses the popular `FastAPI `_ and `Pydantic `_ frameworks under the hood. This means you can use all their features while building your App. - -pydantic enables fast data validation and settings management using Python type annotations and FastAPI is a modern, fast (high-performance), web framework for building APIs. - -You can easily use pydantic by defining your own payload format. - -.. literalinclude:: models.py - -Then, type your handler input with your custom model. - -.. literalinclude:: post_example_pydantic.py - -After running the updated App, the App documentation ``/name`` has changed and takes JSON with ``{"name": ...}`` as input. - -.. figure:: https://pl-public-data.s3.amazonaws.com/assets_lightning/rest_post_pydantic.png - :alt: Rest API with pydantic - :width: 100 % - -You can invoke the RESTful API route ``/name`` with the following command: - -.. code-block:: bash - - curl -X 'POST' \ - 'http://127.0.0.1:7501/name' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "my_name" - }' - -.. note:: - - Using curl, you can pass a JSON payload using the ``-d`` argument. - ----- - -********** -Learn more -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add an API Route to your App - :description: Learn how to develop a simple API for your App. - :col_css: col-md-6 - :button_link: add_api.html - :height: 150 - -.. displayitem:: - :header: Develop a Command Line Interface (CLI) - :description: Learn how to develop an CLI for your App. - :col_css: col-md-6 - :button_link: ../build_command_line_interface/index.html - :height: 150 - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/debug_locally.rst b/docs/source-app/workflows/debug_locally.rst deleted file mode 100644 index cd5a5a80fde7c..0000000000000 --- a/docs/source-app/workflows/debug_locally.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -##################################### -Debug a Distributed Cloud App Locally -##################################### diff --git a/docs/source-app/workflows/enable_fault_tolerance.rst b/docs/source-app/workflows/enable_fault_tolerance.rst deleted file mode 100644 index b1630d4d396ac..0000000000000 --- a/docs/source-app/workflows/enable_fault_tolerance.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -###################### -Enable Fault Tolerance -###################### diff --git a/docs/source-app/workflows/extend_app.rst b/docs/source-app/workflows/extend_app.rst deleted file mode 100644 index bc7ffd7d2f87a..0000000000000 --- a/docs/source-app/workflows/extend_app.rst +++ /dev/null @@ -1,59 +0,0 @@ -###################### -Extend an Existing App -###################### -You can extend a Lightning App by using community components or building your own. - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Add more Components - :description: Extend an App by adding a prebuilt component. - :col_css: col-md-4 - :button_link: add_components.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Add a web user interface (UI) - :description: Extend an App by adding a web user interface (UI) - :col_css: col-md-4 - :button_link: add_web_ui/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Add a URL link - :description: Extend an App by adding a web URL link - :col_css: col-md-4 - :button_link: add_web_link.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Build a Component - :description: Extend an App by building a Component - :col_css: col-md-4 - :button_link: build_lightning_component/index.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Add a server - :description: Extend an App by adding a server to a Component. - :col_css: col-md-4 - :button_link: add_server/index.html - :height: 150 - :tag: Intermediate - - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/index.rst b/docs/source-app/workflows/index.rst deleted file mode 100644 index 9993971873286..0000000000000 --- a/docs/source-app/workflows/index.rst +++ /dev/null @@ -1,186 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :hidden: - - access_app_state - add_web_ui/index - add_web_link - secrets <../glossary/secrets> - arrange_tabs/index - connect components <../levels/intermediate/connect_lightning_components> - build components <../levels/basic/build_a_lightning_component> - run_work_once - cloud compute <../core_api/lightning_work/compute> - build_command_line_interface/index - rest API <../glossary/restful_api/restful_api> - extend_app - build_lightning_component/publish_a_component - add_server/index - run_app_on_cloud/index - run_work_in_parallel - drive <../glossary/storage/drive> - share_app - share_files_between_components - -####### -How to: -####### - -.. raw:: html - -
-
- -.. displayitem:: - :header: Access the App State - :description: Learn to work with the app state - :col_css: col-md-4 - :button_link: access_app_state.html - :height: 180 - -.. displayitem:: - :header: Add a web user interface - :description: Learn how to add React, StreamLit, Dash to your App. - :col_css: col-md-4 - :button_link: add_web_ui/index.html - :height: 180 - -.. displayitem:: - :header: Add a web link - :description: Learn how to embed external websites - :col_css: col-md-4 - :button_link: add_web_link.html - :height: 180 - -.. displayitem:: - :header: Add encrypted secrets - :description: Learn how to organize your UI - :col_css: col-md-4 - :button_link: ../glossary/secrets.html - :height: 180 - -.. displayitem:: - :header: Arrange App tabs - :description: Learn how to organize your UI - :col_css: col-md-4 - :button_link: arrange_tabs/index.html - :height: 180 - -.. displayitem:: - :header: Build a Lightning App - :description: Simple App to get started - :col_css: col-md-4 - :button_link: ../levels/basic/connect_lightning_components.html - :height: 180 - -.. displayitem:: - :header: Build a Lightning Component - :description: Understand how to separated the glue from the actual work - :col_css: col-md-4 - :button_link: ../levels/basic/build_a_lightning_component.html - :height: 180 - -.. displayitem:: - :header: Cache Work run calls - :description: Understand how to trigger a work run method - :col_css: col-md-4 - :button_link: run_work_once.html - :height: 180 - -.. displayitem:: - :header: Customize your cloud compute - :description: Select machines to run on - :col_css: col-md-4 - :button_link: ../core_api/lightning_work/compute.html - :height: 180 - -.. displayitem:: - :header: Develop a Command Line Interface (CLI) - :description: Learn to develop a CLI - :col_css: col-md-4 - :button_link: build_command_line_interface/index.html - :height: 180 - -.. displayitem:: - :header: Develop a Lightning App - :description: Learn to connect components together into a Lightning App - :col_css: col-md-4 - :button_link: ../levels/basic/connect_lightning_components.html - :height: 180 - -.. displayitem:: - :header: Develop a REST API - :description: Learn to deploy a model behind a REST API - :col_css: col-md-4 - :button_link: ../glossary/restful_api/restful_api.html - :height: 180 - -.. displayitem:: - :header: Extend an existing App - :description: Learn where to go next with an App - :col_css: col-md-4 - :button_link: extend_app.html - :height: 180 - -.. displayitem:: - :header: Publish a Lightning Component - :description: Share your components with others - :col_css: col-md-4 - :button_link: build_lightning_component/publish_a_component.html - :height: 180 - -.. displayitem:: - :header: Run a server within a Lightning App - :description: Lightning Work can be infinite jobs - :col_css: col-md-4 - :button_link: add_server/index.html - :height: 180 - -.. displayitem:: - :header: Run an App on the cloud - :description: Learn how to get things done in the cloud with ease - :col_css: col-md-4 - :button_link: run_app_on_cloud/index.html - :height: 180 - -.. displayitem:: - :header: Run Works in parallel - :description: Learn how to make your Work non blocking - :col_css: col-md-4 - :button_link: run_work_in_parallel.html - :height: 180 - -.. displayitem:: - :header: Save files - :description: Learn how to save files in a work by using Drive - :col_css: col-md-4 - :button_link: ../glossary/storage/drive.html - :height: 180 - -.. displayitem:: - :header: Share an App - :description: Learn how to share your work with others - :col_css: col-md-4 - :button_link: share_app.html - :height: 180 - -.. displayitem:: - :header: Share files between components - :description: Learn how Lightning Storage emulates a single filesystem in a distributed setting - :col_css: col-md-4 - :button_link: share_files_between_components.html - :height: 180 - -.. displayitem:: - :header: Mount Cloud Data - :description: Learn how Lightning Mounts are used to make the contents of an cloud object store bucket available on disk when running in the cloud. - :col_css: col-md-4 - :button_link: mount_cloud_object_store.html - :height: 180 - - - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/mount_cloud_object_store.rst b/docs/source-app/workflows/mount_cloud_object_store.rst deleted file mode 100644 index 31ac2a44aa3c0..0000000000000 --- a/docs/source-app/workflows/mount_cloud_object_store.rst +++ /dev/null @@ -1,141 +0,0 @@ -:orphan: - -############## -Add Cloud Data -############## - -**Audience:** Users who want to read files stored in a Cloud Object Bucket in an app. - -****************************** -Mounting Public AWS S3 Buckets -****************************** - -=================== -Add Mount to a Work -=================== - -To mount data from a cloud bucket to your app compute, initialize a :class:`~lightning.app.storage.mount.Mount` -object with the source path of the s3 bucket and the absolute directory path where it should be mounted and -pass the :class:`~lightning.app.storage.mount.Mount` to the :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute` -of the :class:`~lightning.app.core.work.LightningWork` it should be mounted on. - -In this example, we will mount an S3 bucket: ``s3://ryft-public-sample-data/esRedditJson/`` to ``/content/esRedditJson/``. - -.. code-block:: python - - from lightning.app import CloudCompute - from lightning.app.storage import Mount - - self.my_work = MyWorkClass( - cloud_compute=CloudCompute( - mounts=Mount( - source="s3://ryft-public-sample-data/esRedditJson/", - mount_path="/content/esRedditJson/", - ), - ) - ) - -You can also pass multiple mounts to a single work by passing a ``List[Mount(...), ...]`` to the -``CloudCompute(mounts=...)`` argument. - -.. note:: - - * Mounts supported up to 1 Million files, 5GB per file. Need larger mounts? Contact support@lightning.ai - * When adding multiple mounts, each one should have a unique ``mount_path``. - * A maximum of 10 :class:`~lightning.app.storage.mount.Mount`\s can be added to a :class:`~lightning.app.core.work.LightningWork`. - -======================= -Read Files From a Mount -======================= - -Once a :class:`~lightning.app.storage.mount.Mount` object is passed to :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute`, -you can access, list, or read any file from the mount under the specified ``mount_path``, just like you would if it -was on your local machine. - -Assuming your ``mount_path`` is ``"/content/esRedditJson/"`` you can do the following: - ----------- -Read Files ----------- - -.. code-block:: python - - with open("/content/esRedditJson/esRedditJson1", "r") as f: - some_data = f.read() - - # do something with "some_data"... - ----------- -List Files ----------- - -.. code-block:: python - - files = os.listdir("/content/esRedditJson/") - --------------------- -See the Full Example --------------------- - -.. code-block:: python - :emphasize-lines: 10,15 - - import os - - import lightning as L - from lightning.app import CloudCompute - from lightning.app.storage import Mount - - class ReadMount(L.LightningWork): - def run(self): - # Print a list of files stored in the mounted S3 Bucket. - files = os.listdir("/content/esRedditJson/") - for file in files: - print(file) - - # Read the contents of a particular file in the bucket "esRedditJson1" - with open("/content/esRedditJson/esRedditJson1", "r") as f: - some_data = f.read() - # do something with "some_data"... - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.my_work = ReadMount( - cloud_compute=CloudCompute( - mounts=Mount( - source="s3://ryft-public-sample-data/esRedditJson/", - mount_path="/content/esRedditJson/", - ), - ) - ) - - def run(self): - self.my_work.run() - -.. note:: - - When running a Lightning App on your local machine, any :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute` - configuration (including a :class:`~lightning.app.storage.mount.Mount`) is ignored at runtime. If you need access to - these files on your local disk, you should download a copy of them to your machine. - -.. note:: - - Mounted files from an S3 bucket are ``read-only``. Any modifications, additions, or deletions - to files in the mounted directory will not be reflected in the cloud object store. - ----- - -********************************************** -Mounting Private AWS S3 Buckets - Coming Soon! -********************************************** - -We'll Let you know when this feature is ready! - ----- - -************************************************ -Mounting Google Cloud GCS Buckets - Coming Soon! -************************************************ - -We'll Let you know when this feature is ready! diff --git a/docs/source-app/workflows/run_app_on_cloud/cloud_files.rst b/docs/source-app/workflows/run_app_on_cloud/cloud_files.rst deleted file mode 100644 index 8e8f01a00f3d7..0000000000000 --- a/docs/source-app/workflows/run_app_on_cloud/cloud_files.rst +++ /dev/null @@ -1,69 +0,0 @@ -.. _ignore: - -################################## -Configure Your Lightning Cloud App -################################## - -**Audience:** Users who want to control Lightning App files on the cloud. - ----- - -************************************** -Ignore file uploads to Lightning cloud -************************************** -Running Lightning Apps on the cloud will upload the source code of your app to the cloud. You can use ``.lightningignore`` file(s) to ignore files or directories while uploading. The `.lightningignore` file follows the same format as a `.gitignore` -file. - -For example, the source code directory below with the ``.lightningignore`` file will ignore the file named -``model.pt`` and directory named ``data_dir``. - -.. code:: bash - - . - ├── README.md - ├── app.py - ├── data_dir - │ ├── image1.png - │ ├── image2.png - │ └── ... - ├── .lightningignore - ├── requirements.txt - └── model.pt - -.. code:: bash - - ~/project/home ❯ cat .lightningignore - model.pt - data_dir - -A sample ``.lightningignore`` file can be found `here `_. - -If you are a component author and your components creates local files that you want to ignore, you can do: - -.. code-block:: python - - class MyComponent(L.LightningWork): # or L.LightningFlow - def __init__(self): - super().__init__() - self.lightningignore = ("model.pt", "data_dir") - - -This has the benefit that the files will be ignored automatically for all the component users, making an easier -transition between running locally vs in the cloud. - ----- - -******************* -Structure app files -******************* - -We recommend your app contain the following files: - -.. code:: bash - - . - ├── .lightning (auto-generated- contains Lightning configuration) - ├── .lightningignore (contains files not to upload to the cloud) - ├── app.py - ├── README.md (optional- a markdown description of your app) - └── requirements.txt (optional- contains all your app dependencies) diff --git a/docs/source-app/workflows/run_app_on_cloud/index.rst b/docs/source-app/workflows/run_app_on_cloud/index.rst deleted file mode 100644 index 55bc3b6807809..0000000000000 --- a/docs/source-app/workflows/run_app_on_cloud/index.rst +++ /dev/null @@ -1,5 +0,0 @@ -##################### -Run apps on the cloud -##################### - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/run_app_on_cloud/index_content.rst b/docs/source-app/workflows/run_app_on_cloud/index_content.rst deleted file mode 100644 index 737b4365df4e2..0000000000000 --- a/docs/source-app/workflows/run_app_on_cloud/index_content.rst +++ /dev/null @@ -1,115 +0,0 @@ -.. _run_app_in_cloud: - -.. toctree:: - :maxdepth: 1 - :hidden: - - cloud_files - lightning_cloud - on_prem - on_your_own_machine - -**Audience:** Users who want to share or scale Lightning Apps. - ----- - -***************************** -Run on Lightning Public Cloud -***************************** - -You can run Lightning Apps for free on the Public Lightning cloud with a single flag! - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Run on Lightning Cloud - :description: Learn how to run on the Lightning public cloud - :col_css: col-md-4 - :button_link: lightning_cloud.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Choose Hardware - :description: Configure you app cloud resources - :col_css: col-md-4 - :button_link: ../../core_api/lightning_work/compute.html - :height: 150 - :tag: Basic - -.. displayitem:: - :header: Set Environment Variables - :description: Manage your environment variables in the cloud - :col_css: col-md-4 - :button_link: ../../glossary/environment_variables.html - :height: 150 - :tag: Basic - -.. displayitem:: - :header: Configure Your Lightning Cloud App - :description: Customize your cloud apps files - :col_css: col-md-4 - :button_link: cloud_files.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Manage App Dependencies - :description: Configure your python requirements or use a custom docker image - :col_css: col-md-4 - :button_link: ../../glossary/build_config/build_config.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Share Files Between Works - :description: Learn more about data transferring - :col_css: col-md-4 - :button_link: ../../glossary/storage/storage.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
- ----- - -************ -Other Clouds -************ - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Run On Your Own Machine - :description: Run Lightning Apps on any machine - :col_css: col-md-4 - :button_link: on_your_own_machine.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Run On Your Private Cloud - :description: Run Lightning Apps on your own cloud - :col_css: col-md-4 - :button_link: on_prem.html - :height: 150 - :tag: basic - - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst b/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst deleted file mode 100644 index 494a523852bd9..0000000000000 --- a/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst +++ /dev/null @@ -1,67 +0,0 @@ -####################### -Run an App on the Cloud -####################### - -**Audience:** Users who want to share their apps or run on specialized hardware (like GPUs). - ----- - -********************************* -Run on the public Lightning cloud -********************************* -To run any app on the public lightning cloud use the ``--cloud`` argument: - -.. code:: bash - - lightning_app run app app.py --cloud - - -.. note:: - By default, running your apps on the public Lightning cloud is free of charge using default CPUs, and any app uploaded to the Lightning cloud will be shared with the community (source code and app view will be public). If you would like to make your apps private please `contact us `_. - -If your app contains ``LightningWork`` components that require more compute resources, such as larger CPUs or **GPUs**, you'll need to add credits to your Lightning AI account. - - ----- - -************************** -Add dependencies to my app -************************** - - -Add all dependencies required to run your app to a `requirements.txt` file in your app's directory. Read :ref:`build_config` for more details. - - - ----- - - -******** -Name app -******** - -Simply use the ``--name`` flag when running your app, for example: - -.. code:: bash - - lightning_app run app app.py --cloud --name my-awesome-app - -Alternatively, you can change the name of the app in the ``.lightning`` file: - -.. code:: bash - - ~/project/home ❯ cat .lightning - name: my-awesome-app - -The ``.lightning`` file is a general configuration file. -To learn more about optional configuration file parameters, see :class:`~lightning.app.utilities.packaging.app_config.AppConfig`. - ------- - -******************** -Choose Cloud Compute -******************** - -You can configure the hardware your app is running on by setting a :class:`~lightning.app.utilities.packaging.cloud_compute.CloudCompute` object onto the ``cloud_compute`` property of your work's. - -Learn more with the :ref:`cloud_compute` guide diff --git a/docs/source-app/workflows/run_app_on_cloud/on_prem.rst b/docs/source-app/workflows/run_app_on_cloud/on_prem.rst deleted file mode 100644 index be0a954f29b16..0000000000000 --- a/docs/source-app/workflows/run_app_on_cloud/on_prem.rst +++ /dev/null @@ -1,6 +0,0 @@ -########################### -Run an App on Private Cloud -########################### - - -To run Lightning apps on a private or on-prem cluster, `contact us `_. diff --git a/docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst b/docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst deleted file mode 100644 index 8226a1a00469b..0000000000000 --- a/docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst +++ /dev/null @@ -1,26 +0,0 @@ -####################### -Run on your own machine -####################### - -**Audience:** Users who want to run Lightning App on a remote machine. - ----- - -*********** -Run via ssh -*********** -To run a Lightning App on any machine, simply ssh to the machine and run the app directly - -.. code:: bash - - # Copy over credentials from your local machine to your cloud machine - scp ~/.lightning/credentials.json your_name@your_cloud_machine:~/.lightning - - # log into your cloud machine - ssh your_name@your_cloud_machine - - # get your code on the machine and install deps - ... - - # start the app - lightning run app app.py diff --git a/docs/source-app/workflows/run_app_snippet.rst b/docs/source-app/workflows/run_app_snippet.rst deleted file mode 100644 index 6283cb88104eb..0000000000000 --- a/docs/source-app/workflows/run_app_snippet.rst +++ /dev/null @@ -1,33 +0,0 @@ -:orphan: - -*********** -Run the app -*********** - -.. raw:: html - -
-
- -Run the app with the ``run`` command - -.. code:: bash - - lightning_app run app app.py - -.. raw:: html - -
-
- - -Add the ``--cloud`` argument to run on the `lightning cloud `_. - -.. code:: bash - - lightning_app run app app.py --cloud - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/run_components_on_different_hardware.rst b/docs/source-app/workflows/run_components_on_different_hardware.rst deleted file mode 100644 index 9685c3461e511..0000000000000 --- a/docs/source-app/workflows/run_components_on_different_hardware.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -#################################### -Run components on different hardware -#################################### diff --git a/docs/source-app/workflows/run_on_private_cloud.rst b/docs/source-app/workflows/run_on_private_cloud.rst deleted file mode 100644 index 84d64e9060bde..0000000000000 --- a/docs/source-app/workflows/run_on_private_cloud.rst +++ /dev/null @@ -1,26 +0,0 @@ -:orphan: - -###################### -Run on a private cloud -###################### -**Audience:** Users looking to run Lightning apps on their private cloud accounts. - ----- - -****************************** -Run on a private cloud account -****************************** -For enterprise, startups and University use-cases, Lightning AI can run on your own AWS account (with your own credentials), with all the infrastructure fully managed by us. -To enable this, contact our support team to get started: - -onprem@lightning.ai - ----- - - -*********** -Run on-prem -*********** -For enterprise-level security with full control of the Lightning AI system on your own on-prem cluster, contact our support team to get started: - -onprem@lightning.ai diff --git a/docs/source-app/workflows/run_work_in_parallel.rst b/docs/source-app/workflows/run_work_in_parallel.rst deleted file mode 100644 index b87e653afc920..0000000000000 --- a/docs/source-app/workflows/run_work_in_parallel.rst +++ /dev/null @@ -1,10 +0,0 @@ -############################# -Run LightningWork in parallel -############################# -**Audience:** Users who want to run a LightningWork in parallel (asynchronously). - -**Prereqs:** You must have finished the :doc:`Basic levels <../levels/basic/index>`. - ----- - -.. include:: run_work_in_parallel_content.rst diff --git a/docs/source-app/workflows/run_work_in_parallel_content.rst b/docs/source-app/workflows/run_work_in_parallel_content.rst deleted file mode 100644 index 1c8d5b374dbb2..0000000000000 --- a/docs/source-app/workflows/run_work_in_parallel_content.rst +++ /dev/null @@ -1,41 +0,0 @@ - - - -************************************ -When to run a Components in parallel -************************************ -Run LightningWork in parallel when you want to execute work in the background or at the same time as another work. -An example of when this comes up in machine learning is when data streams-in while a model trains. - ----- - -************ -Toy example -************ -By default, a Component must complete before the next one runs. We can enable one -component to start in parallel which allows the code to proceed without having -to wait for the first one to finish. - -.. lit_tabs:: - :descriptions: No parallel components; Allow the train component to run in parallel; When the component runs, it will run in parallel; The next component is unblocked and can now immediately run. - :code_files: /workflows/scripts/parallel/toy_app.py; /workflows/scripts/parallel/toy_parallel.py; /workflows/scripts/parallel/toy_parallel.py; /workflows/scripts/parallel/toy_parallel.py; - :highlights: ; 18; 23; 24; - :enable_run: true - :tab_rows: 3 - :height: 540px - ----- - -******************************* -Multiple components in parallel -******************************* -In this example, we start all 3 components at once. The first two start in parallel, which -allows the third component to run without waiting for the others to finish. - -.. lit_tabs:: - :descriptions: No parallel components; Enable 2 components to run in parallel; Start both components together in parallel; Last component is not blocked and can start immediately. - :code_files: /workflows/scripts/parallel/toy_two_parallel_not_started.py; /workflows/scripts/parallel/toy_two_parallel.py; /workflows/scripts/parallel/toy_two_parallel.py; /workflows/scripts/parallel/toy_two_parallel.py - :highlights: ; 18, 19; 23, 24; 25 - :enable_run: true - :tab_rows: 3 - :height: 540px diff --git a/docs/source-app/workflows/run_work_once.rst b/docs/source-app/workflows/run_work_once.rst deleted file mode 100644 index 8bdd576a2bc76..0000000000000 --- a/docs/source-app/workflows/run_work_once.rst +++ /dev/null @@ -1,13 +0,0 @@ -########################## -Cache LightningWork Runs -########################## - -**Audience:** Users who want to know how ``LightningWork`` works. - -**Level:** Advanced - -**Prereqs**: Level 16+ and read the :doc:`Event Loop guide <../glossary/event_loop>`. - ----- - -.. include:: run_work_once_content.rst diff --git a/docs/source-app/workflows/run_work_once_content.rst b/docs/source-app/workflows/run_work_once_content.rst deleted file mode 100644 index 355c2ed2bac43..0000000000000 --- a/docs/source-app/workflows/run_work_once_content.rst +++ /dev/null @@ -1,151 +0,0 @@ - -******************************************************** -What caching the calls of Work's run method does for you -******************************************************** - -By default, the run method in a LightningWork (Work) "remembers" (caches) the input arguments it is getting called with and does not execute again if called with the same arguments again. -In other words, the run method only executes when the input arguments have never been seen before. - -You can turn caching on or off: - -.. code-block:: python - - # Run only when the input arguments change (default) - work = MyWork(cache_calls=True) - - # Run every time regardless of whether input arguments change or not - work = MyWork(cache_calls=False) - -To better understand this, imagine that every day you want to sequentially download and process some data and then train a model on that data. -As explained in the :doc:`Event Loop guide <../../glossary/event_loop>`, the Lightning App runs within an infinite while loop, so the pseudo-code of your application might looks like this: - -.. code-block:: python - - from datetime import datetime - - # Lightning code - while True: # This is the Lightning Event Loop - - # Your code - today = datetime.now().strftime("%D") # '05/25/22' - data_processor.run(today) - train_model.run(data_processor.data) - -In this scenario, you want your components to run ``once`` a day, and no more than that! But your code is running within an infinite loop, how can this even work? -This is where the Work's internal caching mechanism comes in. By default, Lightning caches a hash of the input provided to its run method and won't re-execute the method if the same input is provided again. -In the example above, the **data_processor** component run method receives the string **"05/25/22"**. It runs one time and any further execution during the day is skipped until tomorrow is reached and the work run method receives **06/25/22**. This logic applies everyday. -This caching mechanism is inspired from how `React.js Components and Props `_ renders websites. Only changes to the inputs re-trigger execution. - -*************** -Caching Example -*************** - -Here's an example of this behavior with LightningWork: - -.. code:: python - :emphasize-lines: 11, 17 - - import lightning as L - - - class ExampleWork(L.LightningWork): - def run(self, *args, **kwargs): - print(f"I received the following props: args: {args} kwargs: {kwargs}") - - - work = ExampleWork() - work.run(value=1) - - # Providing the same value. This won't run as already cached. - work.run(value=1) - work.run(value=1) - work.run(value=1) - work.run(value=1) - - # Changing the provided value. This isn't cached and will run again. - work.run(value=10) - -And you should see the following by running the code above: - -.. code-block:: console - - $ python example.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 10} - -As you can see, the intermediate run didn't execute, as we would expected when ``cache_calls=True``. - -*********************************** -Implications of turning caching off -*********************************** - -By setting ``cache_calls=False``, Lightning won't cache the return value and re-execute the run method on every call. - -.. code:: python - :emphasize-lines: 7 - - from lightning.app import LightningWork - - - class ExampleWork(LightningWork): - def run(self, *args, **kwargs): - print(f"I received the following props: args: {args} kwargs: {kwargs}") - - - work = ExampleWork(cache_calls=False) - work.run(value=1) - - # Providing the same value. This won't run as already cached. - work.run(value=1) - work.run(value=1) - work.run(value=1) - work.run(value=1) - - # Changing the provided value. This isn't cached and will run again. - work.run(value=10) - -.. code-block:: console - - $ python example.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 10} - -Be aware than when setting both ``cache_calls=False`` and ``parallel=False`` to a work, the code after the ``self.work.run()`` is unreachable -as the work continuously execute in a blocking way. - -.. code-block:: python - :emphasize-lines: 9-10 - - from lightning.app import LightningApp, LightningFlow, LightningWork - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - - self.work = Work(cache_calls=False, parallel=False) - - def run(self): - print("HERE BEFORE") - self.work.run() - print("HERE AFTER") - - - app = LightningApp(Flow()) - -.. code-block:: console - - $ lightning run app app.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - print("HERE BEFORE") - print("HERE BEFORE") - print("HERE BEFORE") - ... diff --git a/docs/source-app/workflows/schedule_apps.rst b/docs/source-app/workflows/schedule_apps.rst deleted file mode 100644 index 7b596cd08b179..0000000000000 --- a/docs/source-app/workflows/schedule_apps.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -################# -Schedule App Runs -################# diff --git a/docs/source-app/workflows/scripts/parallel/toy_app.py b/docs/source-app/workflows/scripts/parallel/toy_app.py deleted file mode 100644 index 8cfcd61c5bd31..0000000000000 --- a/docs/source-app/workflows/scripts/parallel/toy_app.py +++ /dev/null @@ -1,27 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class AnalyzeComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class LitWorkflow(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('cpu')) - - - def run(self): - self.train.run("machine A counting") - self.analyze.run("machine B counting") - - -app = LightningApp(LitWorkflow()) diff --git a/docs/source-app/workflows/scripts/parallel/toy_parallel.py b/docs/source-app/workflows/scripts/parallel/toy_parallel.py deleted file mode 100644 index 6b4059caa4209..0000000000000 --- a/docs/source-app/workflows/scripts/parallel/toy_parallel.py +++ /dev/null @@ -1,27 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class AnalyzeComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class LitWorkflow(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu'), parallel=True) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('cpu')) - - - def run(self): - self.train.run("machine A counting") - self.analyze.run("machine B counting") - - -app = LightningApp(LitWorkflow()) diff --git a/docs/source-app/workflows/scripts/parallel/toy_two_parallel.py b/docs/source-app/workflows/scripts/parallel/toy_two_parallel.py deleted file mode 100644 index 57967137f0f25..0000000000000 --- a/docs/source-app/workflows/scripts/parallel/toy_two_parallel.py +++ /dev/null @@ -1,27 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class AnalyzeComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class LitWorkflow(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu'), parallel=True) - self.baseline_1 = TrainComponent(cloud_compute=CloudCompute('cpu'), parallel=True) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('cpu')) - - def run(self): - self.train.run("machine A counting") - self.baseline_1.run("machine C counting") - self.analyze.run("machine B counting") - -app = LightningApp(LitWorkflow()) diff --git a/docs/source-app/workflows/scripts/parallel/toy_two_parallel_not_started.py b/docs/source-app/workflows/scripts/parallel/toy_two_parallel_not_started.py deleted file mode 100644 index 7ded9d8a93935..0000000000000 --- a/docs/source-app/workflows/scripts/parallel/toy_two_parallel_not_started.py +++ /dev/null @@ -1,27 +0,0 @@ -# app.py -from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute - - -class TrainComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class AnalyzeComponent(LightningWork): - def run(self, message): - for i in range(100000000000): - print(message, i) - -class LitWorkflow(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.train = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.baseline_1 = TrainComponent(cloud_compute=CloudCompute('cpu')) - self.analyze = AnalyzeComponent(cloud_compute=CloudCompute('cpu')) - - def run(self): - self.train.run("machine A counting") - self.baseline_1.run("machine C counting") - self.analyze.run("machine B counting") - -app = LightningApp(LitWorkflow()) diff --git a/docs/source-app/workflows/share_app.rst b/docs/source-app/workflows/share_app.rst deleted file mode 100644 index 8d7482e21d2c3..0000000000000 --- a/docs/source-app/workflows/share_app.rst +++ /dev/null @@ -1,33 +0,0 @@ -############ -Share an App -############ -**Audience:** Users who want to show off their work. - ----- - -*********************************** -Option 1: Run on the cloud to share -*********************************** -To share an app, simply run your app on the cloud: - -.. code:: bash - - lightning run app app.py --cloud - -Then share the link that's generated. - ----- - -********************************** -Option 2: Expose a tunnel to share -********************************** -If you'd like to share yourself, feel free to run the app in local mode -and expose the URlapp. - -Run local: - -.. code:: bash - - lightning run app app.py - -And then, use one of the many guides to `expose a tunnel `_. diff --git a/docs/source-app/workflows/share_files_between_components.rst b/docs/source-app/workflows/share_files_between_components.rst deleted file mode 100644 index 5ccf2b9e2a441..0000000000000 --- a/docs/source-app/workflows/share_files_between_components.rst +++ /dev/null @@ -1,120 +0,0 @@ -:orphan: - -############################## -Share Files Between Components -############################## - -.. note:: The contents of this page is still in progress! - -**Audience:** Users who want to share files between components. - ----- - -********************************** -Why do I need distributed storage? -********************************** -In a Lightning App some components can be executed on their own hardware. Distributed storage -enables a file saved by a component on one machine to be used by components in other machines (transparently). - -If you've asked the question "how do I use the checkpoint from this model to deploy this other thing", you've -needed distributed storage. - ----- - -************ -Write a file -************ -To write a file, first create a reference to the file with the :class:`~lightning.app.storage.path.Path` class, then write to it: - -.. code:: python - - from lightning.app.storage import Path - - # file reference - boring_file_reference = Path("boring_file.txt") - - # write to that file - with open(self.boring_file_reference, "w") as f: - f.write("yolo") - - ----- - -********** -Use a file -********** -To use a file, pass the reference to the file: - -.. code:: python - - f = open(boring_file_reference, "r") - print(f.read()) - ----- - -.. - ******************************** - Create a directory - coming soon - ******************************** - - - ---- - - ****************************** - Use a directory - coming soon - ****************************** - TODO - - ---- - -********************************* -Example: Share a model checkpoint -********************************* -A common workflow in ML is to use a checkpoint created by another component. -First, define a component that saves a checkpoint: - -.. literalinclude:: ./share_files_between_components/app.py - :lines: -19 - -Next, define a component that needs the checkpoints: - -.. literalinclude:: ./share_files_between_components/app.py - :lines: 20-31 - -Link both components via a parent component: - -.. literalinclude:: ./share_files_between_components/app.py - :lines: 32- - - -Run the app above with the following command: - -.. code-block:: bash - - lightning run app docs/source/workflows/share_files_between_components/app.py - -.. code-block:: console - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - Loaded checkpoint_1: tensor([0, 1, 2, 3, 4]) - Loaded checkpoint_2: tensor([0, 1, 2, 3, 4]) - - -For example, here we save a file on one component and use it in another component: - -.. code:: python - - from lightning.app.storage import Path - - - class ComponentA(LightningWork): - def __init__(self): - super().__init__() - self.boring_path = None - - def run(self): - # This should be used as a REFERENCE to the file. - self.boring_path = Path("boring_file.txt") - with open(self.boring_path, "w") as f: - f.write(FILE_CONTENT) diff --git a/docs/source-app/workflows/share_files_between_components/app.py b/docs/source-app/workflows/share_files_between_components/app.py deleted file mode 100644 index 7bf0686f65954..0000000000000 --- a/docs/source-app/workflows/share_files_between_components/app.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -import torch - -from lightning.app import LightningWork, LightningFlow, LightningApp -from lightning.app.storage.path import Path - - -class ModelTraining(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.checkpoints_path = Path("./checkpoints") - - def run(self): - # make fake checkpoints - checkpoint_1 = torch.tensor([0, 1, 2, 3, 4]) - checkpoint_2 = torch.tensor([0, 1, 2, 3, 4]) - os.makedirs(self.checkpoints_path, exist_ok=True) - checkpoint_path = str(self.checkpoints_path / "checkpoint_{}.ckpt") - torch.save(checkpoint_1, str(checkpoint_path).format("1")) - torch.save(checkpoint_2, str(checkpoint_path).format("2")) - - -class ModelDeploy(LightningWork): - def __init__(self, ckpt_path, *args, **kwargs): - super().__init__() - self.ckpt_path = ckpt_path - - def run(self): - ckpts = os.listdir(self.ckpt_path) - checkpoint_1 = torch.load(os.path.join(self.ckpt_path, ckpts[0])) - checkpoint_2 = torch.load(os.path.join(self.ckpt_path, ckpts[1])) - print(f"Loaded checkpoint_1: {checkpoint_1}") - print(f"Loaded checkpoint_2: {checkpoint_2}") - - -class LitApp(LightningFlow): - def __init__(self): - super().__init__() - self.train = ModelTraining() - self.deploy = ModelDeploy(ckpt_path=self.train.checkpoints_path) - - def run(self): - self.train.run() - self.deploy.run() - - -app = LightningApp(LitApp()) diff --git a/docs/source-app/workflows/test_an_app.rst b/docs/source-app/workflows/test_an_app.rst deleted file mode 100644 index c51ae3aa8f652..0000000000000 --- a/docs/source-app/workflows/test_an_app.rst +++ /dev/null @@ -1,5 +0,0 @@ -:orphan: - -########### -Test an App -########### diff --git a/docs/source-pytorch/versioning.rst b/docs/source-pytorch/versioning.rst index 96038e63cf807..ebae1f920a5a6 100644 --- a/docs/source-pytorch/versioning.rst +++ b/docs/source-pytorch/versioning.rst @@ -16,8 +16,6 @@ A Lightning release number is in the format of ``MAJOR.MINOR.PATCH``. With every release, we publish a changelog where we list additions, removals, deprecations, changed functionality and fixes. -The ``lightning.app`` package is an exception to this rule, as it may contain any change with or without deprecations in any of the releases. - API Stability ************* diff --git a/examples/app/argparse/app.py b/examples/app/argparse/app.py deleted file mode 100644 index 5fa8039908eb3..0000000000000 --- a/examples/app/argparse/app.py +++ /dev/null @@ -1,28 +0,0 @@ -import argparse - -from lightning.app import CloudCompute, LightningApp, LightningFlow, LightningWork - - -class Work(LightningWork): - def __init__(self, cloud_compute): - super().__init__(cloud_compute=cloud_compute) - - def run(self): - pass - - -class Flow(LightningFlow): - def __init__(self, cloud_compute): - super().__init__() - self.work = Work(cloud_compute) - - def run(self): - assert self.work.cloud_compute.name == "gpu", self.work.cloud_compute.name - self.stop() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--use_gpu", action="store_true", default=False, help="Whether to use GPU in the cloud") - hparams = parser.parse_args() - app = LightningApp(Flow(CloudCompute("gpu" if hparams.use_gpu else "cpu"))) diff --git a/examples/app/boring/.gitignore b/examples/app/boring/.gitignore deleted file mode 100644 index 94018704d9f90..0000000000000 --- a/examples/app/boring/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -lightning_logs -*.pt -.storage/ -.shared/ -data -*.ckpt -redis-stable -node_modules -*.rdb -boring_file.txt diff --git a/examples/app/boring/app.py b/examples/app/boring/app.py deleted file mode 100644 index 0dfaedfae0107..0000000000000 --- a/examples/app/boring/app.py +++ /dev/null @@ -1,61 +0,0 @@ -import os - -from lightning.app import CloudCompute, LightningApp, LightningFlow, LightningWork -from lightning.app.components import TracerPythonScript -from lightning.app.storage.path import Path - -FILE_CONTENT = """ -Hello there! -This tab is currently an IFrame of the FastAPI Server running in `DestinationFileAndServeWork`. -Also, the content of this file was created in `SourceFileWork` and then transferred to `DestinationFileAndServeWork`. -Are you already 🤯 ? Stick with us, this is only the beginning. Lightning is 🚀. -""" - - -class SourceFileWork(LightningWork): - def __init__(self, cloud_compute: CloudCompute = CloudCompute(), **kwargs): - super().__init__(parallel=True, **kwargs, cloud_compute=cloud_compute) - self.boring_path = None - - def run(self): - # This should be used as a REFERENCE to the file. - self.boring_path = "lit://boring_file.txt" - with open(self.boring_path, "w", encoding="utf-8") as f: - f.write(FILE_CONTENT) - - -class DestinationFileAndServeWork(TracerPythonScript): - def run(self, path: Path): - assert path.exists() - self.script_args += [f"--filepath={path}", f"--host={self.host}", f"--port={self.port}"] - super().run() - - -class BoringApp(LightningFlow): - def __init__(self): - super().__init__() - self.source_work = SourceFileWork() - self.dest_work = DestinationFileAndServeWork( - script_path=os.path.join(os.path.dirname(__file__), "scripts/serve.py"), - port=1111, - parallel=False, # runs until killed. - cloud_compute=CloudCompute(), - raise_exception=True, - ) - - @property - def ready(self) -> bool: - return self.dest_work.is_running - - def run(self): - self.source_work.run() - if self.source_work.has_succeeded: - # the flow passes the file from one work to another. - self.dest_work.run(self.source_work.boring_path) - self.stop("Boring App End") - - def configure_layout(self): - return {"name": "Boring Tab", "content": self.dest_work.url + "/file"} - - -app = LightningApp(BoringApp()) diff --git a/examples/app/boring/app_dynamic.py b/examples/app/boring/app_dynamic.py deleted file mode 100644 index b08b8cf5ce10d..0000000000000 --- a/examples/app/boring/app_dynamic.py +++ /dev/null @@ -1,72 +0,0 @@ -import os - -from lightning.app import CloudCompute, LightningApp, LightningFlow, LightningWork -from lightning.app.components import TracerPythonScript -from lightning.app.storage.path import Path -from lightning.app.structures import Dict - -FILE_CONTENT = """ -Hello there! -This tab is currently an IFrame of the FastAPI Server running in `DestinationFileAndServeWork`. -Also, the content of this file was created in `SourceFileWork` and then transferred to `DestinationFileAndServeWork`. -Are you already 🤯 ? Stick with us, this is only the beginning. Lightning is 🚀. -""" - - -class SourceFileWork(LightningWork): - def __init__(self, cloud_compute: CloudCompute = CloudCompute(), **kwargs): - super().__init__(parallel=True, **kwargs, cloud_compute=cloud_compute) - self.boring_path = None - - def run(self): - # This should be used as a REFERENCE to the file. - self.boring_path = "lit://boring_file.txt" - with open(self.boring_path, "w") as f: - f.write(FILE_CONTENT) - - -class DestinationFileAndServeWork(TracerPythonScript): - def run(self, path: Path): - assert path.exists() - self.script_args += [f"--filepath={path}", f"--host={self.host}", f"--port={self.port}"] - super().run() - - -class BoringApp(LightningFlow): - def __init__(self): - super().__init__() - self.dict = Dict() - - @property - def ready(self) -> bool: - if "dst_w" in self.dict: - return self.dict["dst_w"].url != "" - return False - - def run(self): - # create dynamically the source_work at runtime - if "src_w" not in self.dict: - self.dict["src_w"] = SourceFileWork() - - self.dict["src_w"].run() - - if self.dict["src_w"].has_succeeded: - # create dynamically the dst_w at runtime - if "dst_w" not in self.dict: - self.dict["dst_w"] = DestinationFileAndServeWork( - script_path=os.path.join(os.path.dirname(__file__), "scripts/serve.py"), - port=1111, - parallel=False, # runs until killed. - cloud_compute=CloudCompute(), - raise_exception=True, - ) - - # the flow passes the file from one work to another. - self.dict["dst_w"].run(self.dict["src_w"].boring_path) - self.stop("Boring App End") - - def configure_layout(self): - return {"name": "Boring Tab", "content": self.dict["dst_w"].url + "/file" if "dst_w" in self.dict else ""} - - -app = LightningApp(BoringApp(), log_level="debug") diff --git a/examples/app/boring/scripts/__init__.py b/examples/app/boring/scripts/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/examples/app/boring/scripts/serve.py b/examples/app/boring/scripts/serve.py deleted file mode 100644 index dedd6013985ca..0000000000000 --- a/examples/app/boring/scripts/serve.py +++ /dev/null @@ -1,29 +0,0 @@ -import argparse -import os - -import uvicorn -from fastapi import FastAPI -from fastapi.requests import Request -from fastapi.responses import HTMLResponse - -if __name__ == "__main__": - parser = argparse.ArgumentParser("Server Parser") - parser.add_argument("--filepath", type=str, help="Where to find the `filepath`") - parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host`") - parser.add_argument("--port", type=int, default="8888", help="Server port`") - hparams = parser.parse_args() - - fastapi_service = FastAPI() - - if not os.path.exists(str(hparams.filepath)): - content = ["The file wasn't transferred"] - else: - with open(hparams.filepath) as fo: - content = fo.readlines() # read the file received from SourceWork. - - @fastapi_service.get("/file") - async def get_file_content(request: Request, response_class=HTMLResponse): - lines = "\n".join(["

" + line + "

" for line in content]) - return HTMLResponse(f"
    {lines}
") - - uvicorn.run(app=fastapi_service, host=hparams.host, port=hparams.port) diff --git a/examples/app/commands_and_api/.lightningignore b/examples/app/commands_and_api/.lightningignore deleted file mode 100644 index f7275bbbd035b..0000000000000 --- a/examples/app/commands_and_api/.lightningignore +++ /dev/null @@ -1 +0,0 @@ -venv/ diff --git a/examples/app/commands_and_api/app.py b/examples/app/commands_and_api/app.py deleted file mode 100644 index 3f59c117c4180..0000000000000 --- a/examples/app/commands_and_api/app.py +++ /dev/null @@ -1,52 +0,0 @@ -from command import CustomCommand, CustomConfig -from lightning import LightningFlow -from lightning.app.api import Get, Post -from lightning.app.core.app import LightningApp - - -async def handler(): - print("Has been called") - return "Hello World !" - - -class ChildFlow(LightningFlow): - def nested_command(self, name: str): - """A nested command.""" - print(f"Hello {name}") - - def configure_commands(self): - return [{"nested_command": self.nested_command}] - - -class FlowCommands(LightningFlow): - def __init__(self): - super().__init__() - self.names = [] - self.child_flow = ChildFlow() - - def run(self): - if self.names: - print(self.names) - - def command_without_client(self, name: str): - """A command without a client.""" - self.names.append(name) - - def command_with_client(self, config: CustomConfig): - self.names.append(config.name) - - def configure_commands(self): - commands = [ - {"command_without_client": self.command_without_client}, - {"command_with_client": CustomCommand(self.command_with_client)}, - ] - return commands + self.child_flow.configure_commands() - - def configure_api(self): - return [ - Post("/user/command_without_client", self.command_without_client), - Get("/pure_function", handler), - ] - - -app = LightningApp(FlowCommands(), log_level="debug") diff --git a/examples/app/commands_and_api/command.py b/examples/app/commands_and_api/command.py deleted file mode 100644 index e2dd26f684b03..0000000000000 --- a/examples/app/commands_and_api/command.py +++ /dev/null @@ -1,18 +0,0 @@ -from argparse import ArgumentParser - -from lightning.app.utilities.commands import ClientCommand -from pydantic import BaseModel - - -class CustomConfig(BaseModel): - name: str - - -class CustomCommand(ClientCommand): - description = "A command with a client." - - def run(self): - parser = ArgumentParser() - parser.add_argument("--name", type=str) - args = parser.parse_args() - self.invoke_handler(config=CustomConfig(name=args.name)) diff --git a/examples/app/components/python/__init__.py b/examples/app/components/python/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/examples/app/components/python/app.py b/examples/app/components/python/app.py deleted file mode 100644 index 944cb7de2995d..0000000000000 --- a/examples/app/components/python/app.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -from pathlib import Path - -from lightning.app import LightningApp, LightningFlow - -from examples.components.python.component_tracer import PLTracerPythonScript - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - script_path = Path(__file__).parent / "pl_script.py" - self.tracer_python_script = PLTracerPythonScript(script_path) - - def run(self): - assert os.getenv("GLOBAL_RANK", "0") == "0" - if not self.tracer_python_script.has_started: - self.tracer_python_script.run() - if self.tracer_python_script.has_succeeded: - self.stop("tracer script succeed") - if self.tracer_python_script.has_failed: - self.stop("tracer script failed") - - -app = LightningApp(RootFlow()) diff --git a/examples/app/components/python/component_popen.py b/examples/app/components/python/component_popen.py deleted file mode 100644 index bc70b9f47b16d..0000000000000 --- a/examples/app/components/python/component_popen.py +++ /dev/null @@ -1,7 +0,0 @@ -from pathlib import Path - -from lightning.app.components import PopenPythonScript - -if __name__ == "__main__": - comp = PopenPythonScript(Path(__file__).parent / "pl_script.py") - comp.run() diff --git a/examples/app/components/python/component_tracer.py b/examples/app/components/python/component_tracer.py deleted file mode 100644 index 3e2e96f38a7f3..0000000000000 --- a/examples/app/components/python/component_tracer.py +++ /dev/null @@ -1,52 +0,0 @@ -from lightning.app.components import TracerPythonScript -from lightning.app.storage.path import Path -from lightning.app.utilities.tracer import Tracer -from lightning.pytorch import Trainer - - -class PLTracerPythonScript(TracerPythonScript): - """This component can be used for ANY PyTorch Lightning script to track its progress and extract its best model - path.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # Define the component state. - self.global_step = None - self.best_model_path = None - - def configure_tracer(self) -> Tracer: - from lightning.pytorch.callbacks import Callback - - class MyInjectedCallback(Callback): - def __init__(self, lightning_work): - self.lightning_work = lightning_work - - def on_train_start(self, trainer, pl_module) -> None: - print("This code doesn't belong to the script but was injected.") - print("Even the Lightning Work is available and state transfer works !") - print(self.lightning_work) - - def on_batch_train_end(self, trainer, *_) -> None: - # On every batch end, collects some information. - # This is communicated automatically to the rest of the app, - # so you can track your training in real time in the Lightning App UI. - self.lightning_work.global_step = trainer.global_step - best_model_path = trainer.checkpoint_callback.best_model_path - if best_model_path: - self.lightning_work.best_model_path = Path(best_model_path) - - # This hook would be called every time - # before a Trainer `__init__` method is called. - - def trainer_pre_fn(trainer, *args, **kwargs): - kwargs["callbacks"] = kwargs.get("callbacks", []) + [MyInjectedCallback(self)] - return {}, args, kwargs - - tracer = super().configure_tracer() - tracer.add_traced(Trainer, "__init__", pre_fn=trainer_pre_fn) - return tracer - - -if __name__ == "__main__": - comp = PLTracerPythonScript(Path(__file__).parent / "pl_script.py") - res = comp.run() diff --git a/examples/app/components/python/pl_script.py b/examples/app/components/python/pl_script.py deleted file mode 100644 index 75538daf4bed2..0000000000000 --- a/examples/app/components/python/pl_script.py +++ /dev/null @@ -1,10 +0,0 @@ -from lightning.pytorch import Trainer -from lightning.pytorch.demos.boring_classes import BoringModel - -if __name__ == "__main__": - model = BoringModel() - trainer = Trainer(max_epochs=1, accelerator="cpu", devices=2, strategy="ddp") - trainer.fit(model) - trainer.validate(model) - trainer.test(model) - trainer.predict(model) diff --git a/examples/app/components/serve/gradio/app.py b/examples/app/components/serve/gradio/app.py deleted file mode 100644 index ec07e4ba99c06..0000000000000 --- a/examples/app/components/serve/gradio/app.py +++ /dev/null @@ -1,51 +0,0 @@ -from functools import partial - -import gradio as gr -import requests -import torch -from lightning.app import LightningApp, LightningFlow -from lightning.app.components import ServeGradio -from PIL import Image - - -# Credit to @akhaliq for his inspiring work. -# Find his original code there: https://huggingface.co/spaces/akhaliq/AnimeGANv2/blob/main/app.py -class AnimeGANv2UI(ServeGradio): - inputs = gr.inputs.Image(type="pil") - outputs = gr.outputs.Image(type="pil") - elon = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/34/Elon_Musk_Royal_Society_%28crop2%29.jpg/330px-Elon_Musk_Royal_Society_%28crop2%29.jpg" - img = Image.open(requests.get(elon, stream=True).raw) - img.save("elon.jpg") - examples = [["elon.jpg"]] - - def __init__(self): - super().__init__() - self.ready = False - - def predict(self, img): - return self.model(img=img) - - def build_model(self): - repo = "AK391/animegan2-pytorch:main" - model = torch.hub.load(repo, "generator", device="cpu") - face2paint = torch.hub.load(repo, "face2paint", size=512, device="cpu") - self.ready = True - return partial(face2paint, model=model) - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.demo = AnimeGANv2UI() - - def run(self): - self.demo.run() - - def configure_layout(self): - tabs = [] - if self.demo.ready: - tabs.append({"name": "Home", "content": self.demo}) - return tabs - - -app = LightningApp(RootFlow()) diff --git a/examples/app/components/serve/gradio/beyonce.jpg b/examples/app/components/serve/gradio/beyonce.jpg deleted file mode 100644 index 68b6084475b01..0000000000000 Binary files a/examples/app/components/serve/gradio/beyonce.jpg and /dev/null differ diff --git a/examples/app/components/serve/gradio/requirements.txt b/examples/app/components/serve/gradio/requirements.txt deleted file mode 100644 index 25aceddaba262..0000000000000 --- a/examples/app/components/serve/gradio/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -gradio diff --git a/examples/app/dag/.gitignore b/examples/app/dag/.gitignore deleted file mode 100644 index fcb9fa9fe4d29..0000000000000 --- a/examples/app/dag/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -df_data -df_target -X_train -X_test -y_train -y_test diff --git a/examples/app/dag/.lightningignore b/examples/app/dag/.lightningignore deleted file mode 100644 index 78ae49041d0b6..0000000000000 --- a/examples/app/dag/.lightningignore +++ /dev/null @@ -1,8 +0,0 @@ -*df_data* -*df_target* -*X_train* -*X_test* -*y_train* -*y_test* -*.shared* -*.storage* diff --git a/examples/app/dag/app.py b/examples/app/dag/app.py deleted file mode 100644 index 7e94f64b62c5d..0000000000000 --- a/examples/app/dag/app.py +++ /dev/null @@ -1,130 +0,0 @@ -import os -from importlib import import_module - -import numpy as np -import pandas as pd -from lightning.app import LightningApp, LightningFlow, LightningWork -from lightning.app.components import TracerPythonScript -from lightning.app.storage import Payload -from lightning.app.structures import Dict, List -from sklearn import datasets -from sklearn.metrics import mean_squared_error - - -def get_path(path): - return os.path.join(os.path.dirname(__file__), path) - - -class GetDataWork(LightningWork): - """This component is responsible to download some data and store them with a PayLoad.""" - - def __init__(self): - super().__init__() - self.df_data = None - self.df_target = None - - def run(self): - print("Starting data collection...") - data = datasets.fetch_california_housing(data_home=get_path("data")) - self.df_data = Payload(pd.DataFrame(data["data"], columns=data["feature_names"])) - self.df_target = Payload(pd.DataFrame(data["target"], columns=["MedHouseVal"])) - print("Finished data collection.") - - -class ModelWork(LightningWork): - """This component is receiving some data and train a sklearn model.""" - - def __init__(self, model_path: str, parallel: bool): - super().__init__(parallel=parallel) - self.model_path, self.model_name = model_path.split(".") - self.test_rmse = None - - def run(self, X_train: Payload, X_test: Payload, y_train: Payload, y_test: Payload): - print(f"Starting training and evaluating {self.model_name}...") - module = import_module(f"sklearn.{self.model_path}") - model = getattr(module, self.model_name)() - model.fit(X_train.value, y_train.value.ravel()) - y_test_prediction = model.predict(X_test.value) - self.test_rmse = np.sqrt(mean_squared_error(y_test.value, y_test_prediction)) - print(f"Finished training and evaluating {self.model_name}.") - - -class DAG(LightningFlow): - """This component is a DAG.""" - - def __init__(self, models_paths: list): - super().__init__() - # Step 1: Create a work to get the data. - self.data_collector = GetDataWork() - - # Step 2: Create a tracer component. This is used to execute python script - # and collect any outputs from its globals as Payloads. - self.processing = TracerPythonScript( - get_path("processing.py"), - outputs=["X_train", "X_test", "y_train", "y_test"], - ) - - # Step 3: Create the work to train the models_paths in parallel. - self.dict = Dict(**{ - model_path.split(".")[-1]: ModelWork(model_path, parallel=True) for model_path in models_paths - }) - - # Step 4: Some element to track components progress. - self.has_completed = False - self.metrics = {} - - def run(self): - # Step 1 and 2: Download and process the data. - self.data_collector.run() - self.processing.run( - df_data=self.data_collector.df_data, - df_target=self.data_collector.df_target, - ) - - # Step 3: Launch n models training in parallel. - for model, work in self.dict.items(): - work.run( - X_train=self.processing.X_train, - X_test=self.processing.X_test, - y_train=self.processing.y_train, - y_test=self.processing.y_test, - ) - if work.test_rmse: # Use the state to control when to collect and stop. - self.metrics[model] = work.test_rmse - work.stop() # Stop the model work to reduce cost - - # Step 4: Print the score of each model when they are all finished. - if len(self.metrics) == len(self.dict): - print(self.metrics) - self.has_completed = True - - -class ScheduledDAG(LightningFlow): - def __init__(self, dag_cls, **dag_kwargs): - super().__init__() - self.dags = List() - self._dag_cls = dag_cls - self.dag_kwargs = dag_kwargs - - def run(self): - """Example of scheduling an infinite number of DAG runs continuously.""" - # Step 1: Every minute, create and launch a new DAG. - if self.schedule("* * * * *"): - print("Launching a new DAG") - self.dags.append(self._dag_cls(**self.dag_kwargs)) - - for dag in self.dags: - if not dag.has_completed: - dag.run() - - -app = LightningApp( - ScheduledDAG( - DAG, - models_paths=[ - "svm.SVR", - "linear_model.LinearRegression", - "tree.DecisionTreeRegressor", - ], - ), -) diff --git a/examples/app/dag/processing.py b/examples/app/dag/processing.py deleted file mode 100644 index 245377fa8cbaa..0000000000000 --- a/examples/app/dag/processing.py +++ /dev/null @@ -1,14 +0,0 @@ -import random - -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import MinMaxScaler - -print("Starting processing ...") -scaler = MinMaxScaler() - -X_train, X_test, y_train, y_test = train_test_split( - df_data.values, df_target.values, test_size=0.20, random_state=random.randint(0, 42) -) -X_train = scaler.fit_transform(X_train) -X_test = scaler.transform(X_test) -print("Finished processing.") diff --git a/examples/app/dag/requirements.txt b/examples/app/dag/requirements.txt deleted file mode 100644 index f669f518e7389..0000000000000 --- a/examples/app/dag/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -scikit-learn -pandas diff --git a/examples/app/display_name/.lightningignore b/examples/app/display_name/.lightningignore deleted file mode 100644 index f7275bbbd035b..0000000000000 --- a/examples/app/display_name/.lightningignore +++ /dev/null @@ -1 +0,0 @@ -venv/ diff --git a/examples/app/display_name/app.py b/examples/app/display_name/app.py deleted file mode 100644 index a3a36b8afb02c..0000000000000 --- a/examples/app/display_name/app.py +++ /dev/null @@ -1,25 +0,0 @@ -from lightning.app import LightningApp, LightningFlow, LightningWork - - -class Work(LightningWork): - def __init__(self, start_with_flow=True): - super().__init__(start_with_flow=start_with_flow) - - def run(self): - pass - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.w = Work() - self.w1 = Work(start_with_flow=False) - self.w.display_name = "My Custom Name" # Not supported yet - self.w1.display_name = "My Custom Name 1" - - def run(self): - self.w.run() - self.w1.run() - - -app = LightningApp(Flow()) diff --git a/examples/app/drive/.gitignore b/examples/app/drive/.gitignore deleted file mode 100644 index eaa5fa8755fc2..0000000000000 --- a/examples/app/drive/.gitignore +++ /dev/null @@ -1 +0,0 @@ -a.txt diff --git a/examples/app/drive/app.py b/examples/app/drive/app.py deleted file mode 100644 index e56636ce13887..0000000000000 --- a/examples/app/drive/app.py +++ /dev/null @@ -1,51 +0,0 @@ -import os - -from lightning.app import LightningApp, LightningFlow, LightningWork -from lightning.app.storage import Drive - - -class Work_1(LightningWork): - def run(self, drive: Drive): - # 1. Create a file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - drive.put("a.txt") - - # 3. Delete the locally. - os.remove("a.txt") - - -class Work_2(LightningWork): - def __init__(self): - super().__init__() - - def run(self, drive: Drive): - print(drive.list(".")) # Prints ["a.txt"] - - print(os.path.exists("a.txt")) # Prints False - - drive.get("a.txt") # Transfer the file from this drive to the local filesystem. - - print(os.path.exists("a.txt")) # Prints True - - with open("a.txt") as f: - print(f.readlines()[0]) # Prints Hello World ! - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - self.work_1 = Work_1() - self.work_2 = Work_2() - - def run(self): - # Pass the drive to both works. - self.work_1.run(self.drive_1) - self.work_2.run(self.drive_1) - self.stop("Application End!") - - -app = LightningApp(Flow()) diff --git a/examples/app/hpo/README.md b/examples/app/hpo/README.md deleted file mode 100644 index b9b648bcaf14f..0000000000000 --- a/examples/app/hpo/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# Build a Lightning Hyperparameter Optimization (HPO) App - -## A bit of background - -Traditionally, developing machine learning (ML) products requires choosing among a large space of -hyperparameters while creating and training the ML models. Hyperparameter optimization -(HPO) aims to find a well-performing hyperparameter configuration for a given ML model -on a dataset at hand, including the ML model, -its hyperparameters, and other data processing steps. - -HPOs free the human expert from a tedious and error-prone, manual hyperparameter tuning process. - -As an example, in the famous [scikit-learn](https://scikit-learn.org/stable/) library, -hyperparameters are passed as arguments to the constructor of -the estimator classes such as `C` kernel for -[Support Vector Classifier](https://scikit-learn.org/stable/modules/classes.html?highlight=svm#module-sklearn.svm), etc. - -It is possible and recommended to search the hyperparameter space for the best validation score. - -An HPO search consists of: - -- an objective method -- a defined parameter space -- a method for searching or sampling candidates - -A naive method for sampling candidates is grid search, which exhaustively considers all -hyperparameter combinations from a user-specified grid. - -Fortunately, HPO is an active area of research, and many methods have been developed to -optimize the time required to get strong candidates. - -In the following tutorial, you will learn how to use Lightning together with [Optuna](https://optuna.org/). - -[Optuna](https://optuna.org/) is an open source HPO framework to automate hyperparameter search. -Out-of-the-box, it provides efficient algorithms to search large spaces and prune unpromising trials for faster results. - -First, you will learn about the best practices on how to implement HPO without the Lightning Framework. -Secondly, we will dive into a working HPO application with Lightning, and finally create a neat -[HiPlot UI](https://facebookresearch.github.io/hiplot/_static/demo/demo_basic_usage.html?hip.filters=%5B%5D&hip.color_by=%22dropout%22&hip.PARALLEL_PLOT.order=%5B%22uid%22%2C%22dropout%22%2C%22lr%22%2C%22loss%22%2C%22optimizer%22%5D) -for our application. - -## Getting started - -### Step 1: Download the data - -```bash -python download_data.py -``` - -### Step 2: Run the HPO Lightning App without an UI - -```bash -lightning run app app_wo_ui.py -``` - -### Step 3: Run the HPO Lightning App with HiPlot UI in Streamlit. - -```bash -lightning run app app_wi_ui.py -``` - -## Learn More - -In the documentation, search for `Build a Sweep App`. diff --git a/examples/app/hpo/app_wi_ui.py b/examples/app/hpo/app_wi_ui.py deleted file mode 100644 index e6ef3f3a1b983..0000000000000 --- a/examples/app/hpo/app_wi_ui.py +++ /dev/null @@ -1,60 +0,0 @@ -from pathlib import Path - -import optuna -from hyperplot import HiPlotFlow -from lightning.app import CloudCompute, LightningApp, LightningFlow -from lightning.app.structures import Dict -from objective import ObjectiveWork - - -class RootHPOFlow(LightningFlow): - def __init__(self, script_path, data_dir, total_trials, simultaneous_trials): - super().__init__() - self.script_path = script_path - self.data_dir = data_dir - self.total_trials = total_trials - self.simultaneous_trials = simultaneous_trials - self.num_trials = simultaneous_trials - self._study = optuna.create_study() - self.ws = Dict() - self.hi_plot = HiPlotFlow() - - def run(self): - if self.num_trials >= self.total_trials: - self.stop() - - has_told_study = [] - - for trial_idx in range(self.num_trials): - work_name = f"objective_work_{trial_idx}" - if work_name not in self.ws: - objective_work = ObjectiveWork( - script_path=self.script_path, - data_dir=self.data_dir, - cloud_compute=CloudCompute("cpu"), - ) - self.ws[work_name] = objective_work - if not self.ws[work_name].has_started: - trial = self._study.ask(ObjectiveWork.distributions()) - self.ws[work_name].run(trial_id=trial._trial_id, **trial.params) - - if self.ws[work_name].metric and not self.ws[work_name].has_told_study: - self.hi_plot.data.append({"x": -1 * self.ws[work_name].metric, **self.ws[work_name].params}) - self._study.tell(self.ws[work_name].trial_id, self.ws[work_name].metric) - self.ws[work_name].has_told_study = True - - has_told_study.append(self.ws[work_name].has_told_study) - - if all(has_told_study): - self.num_trials += self.simultaneous_trials - - -if __name__ == "__main__": - app = LightningApp( - RootHPOFlow( - script_path=str(Path(__file__).parent / "pl_script.py"), - data_dir="data/hymenoptera_data_version_0", - total_trials=6, - simultaneous_trials=2, - ) - ) diff --git a/examples/app/hpo/app_wo_ui.py b/examples/app/hpo/app_wo_ui.py deleted file mode 100644 index e20318347a1f7..0000000000000 --- a/examples/app/hpo/app_wo_ui.py +++ /dev/null @@ -1,57 +0,0 @@ -from pathlib import Path - -import optuna -from lightning.app import CloudCompute, LightningApp, LightningFlow -from lightning.app.structures import Dict -from objective import ObjectiveWork - - -class RootHPOFlow(LightningFlow): - def __init__(self, script_path, data_dir, total_trials, simultaneous_trials): - super().__init__() - self.script_path = script_path - self.data_dir = data_dir - self.total_trials = total_trials - self.simultaneous_trials = simultaneous_trials - self.num_trials = simultaneous_trials - self._study = optuna.create_study() - self.ws = Dict() - - def run(self): - if self.num_trials >= self.total_trials: - self.stop() - - has_told_study = [] - - for trial_idx in range(self.num_trials): - work_name = f"objective_work_{trial_idx}" - if work_name not in self.ws: - objective_work = ObjectiveWork( - script_path=self.script_path, - data_dir=self.data_dir, - cloud_compute=CloudCompute("cpu"), - ) - self.ws[work_name] = objective_work - if not self.ws[work_name].has_started: - trial = self._study.ask(ObjectiveWork.distributions()) - self.ws[work_name].run(trial_id=trial._trial_id, **trial.params) - - if self.ws[work_name].metric and not self.ws[work_name].has_told_study: - self._study.tell(self.ws[work_name].trial_id, self.ws[work_name].metric) - self.ws[work_name].has_told_study = True - - has_told_study.append(self.ws[work_name].has_told_study) - - if all(has_told_study): - self.num_trials += self.simultaneous_trials - - -if __name__ == "__main__": - app = LightningApp( - RootHPOFlow( - script_path=str(Path(__file__).parent / "pl_script.py"), - data_dir="data/hymenoptera_data_version_0", - total_trials=6, - simultaneous_trials=2, - ) - ) diff --git a/examples/app/hpo/download_data.py b/examples/app/hpo/download_data.py deleted file mode 100644 index d82b86a9dee95..0000000000000 --- a/examples/app/hpo/download_data.py +++ /dev/null @@ -1,5 +0,0 @@ -from utils import download_data - -data_dir = "hymenoptera_data_version_0" -download_url = f"https://pl-flash-data.s3.amazonaws.com/{data_dir}.zip" -download_data(download_url, "./data") diff --git a/examples/app/hpo/hyperplot.py b/examples/app/hpo/hyperplot.py deleted file mode 100644 index 8ff238ce38985..0000000000000 --- a/examples/app/hpo/hyperplot.py +++ /dev/null @@ -1,34 +0,0 @@ -from lightning.app import LightningFlow -from lightning.app.frontend import StreamlitFrontend -from lightning.app.utilities.state import AppState - - -class HiPlotFlow(LightningFlow): - def __init__(self): - super().__init__() - self.data = [] - - def run(self): - pass - - def configure_layout(self): - return StreamlitFrontend(render_fn=render_fn) - - -def render_fn(state: AppState): - import json - - import hiplot as hip - import streamlit as st - from streamlit_autorefresh import st_autorefresh - - st.set_page_config(layout="wide") - st_autorefresh(interval=1000, limit=None, key="refresh") - - if not state.data: - st.write("No data available yet ! Stay tuned") - return - - xp = hip.Experiment.from_iterable(state.data) - ret_val = xp.to_streamlit(ret="selected_uids", key="hip").display() - st.markdown("hiplot returned " + json.dumps(ret_val)) diff --git a/examples/app/hpo/objective.py b/examples/app/hpo/objective.py deleted file mode 100644 index e320b66217db1..0000000000000 --- a/examples/app/hpo/objective.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import tempfile -from datetime import datetime -from typing import Optional - -import pandas as pd -import torch -from lightning.app import CloudCompute -from lightning.app.components import TracerPythonScript -from optuna.distributions import CategoricalDistribution, LogUniformDistribution -from torchmetrics import Accuracy - - -class ObjectiveWork(TracerPythonScript): - def __init__(self, script_path: str, data_dir: str, cloud_compute: Optional[CloudCompute]): - timestamp = datetime.now().strftime("%H:%M:%S") - tmpdir = tempfile.TemporaryDirectory().name - submission_path = os.path.join(tmpdir, f"{timestamp}.csv") - best_model_path = os.path.join(tmpdir, f"{timestamp}.model.pt") - super().__init__( - script_path, - script_args=[ - f"--train_data_path={data_dir}/train", - f"--test_data_path={data_dir}/test", - f"--submission_path={submission_path}", - f"--best_model_path={best_model_path}", - ], - cloud_compute=cloud_compute, - ) - self.data_dir = data_dir - self.best_model_path = best_model_path - self.submission_path = submission_path - self.metric = None - self.trial_id = None - self.metric = None - self.params = None - self.has_told_study = False - - def run(self, trial_id: int, **params): - self.trial_id = trial_id - self.params = params - self.script_args.extend([f"--{k}={v}" for k, v in params.items()]) - super().run() - self.compute_metric() - - def _to_labels(self, path: str): - return torch.from_numpy(pd.read_csv(path).label.values) - - def compute_metric(self): - self.metric = -1 * float( - Accuracy(task="binary")( - self._to_labels(self.submission_path), - self._to_labels(f"{self.data_dir}/ground_truth.csv"), - ) - ) - - @staticmethod - def distributions(): - return { - "backbone": CategoricalDistribution(["resnet18", "resnet34"]), - "learning_rate": LogUniformDistribution(0.0001, 0.1), - } diff --git a/examples/app/hpo/pl_script.py b/examples/app/hpo/pl_script.py deleted file mode 100644 index bbc453798431a..0000000000000 --- a/examples/app/hpo/pl_script.py +++ /dev/null @@ -1,43 +0,0 @@ -import argparse -import os - -import pandas as pd -import torch -from flash import Trainer -from flash.image import ImageClassificationData, ImageClassifier - -# Parse arguments provided by the Work. -parser = argparse.ArgumentParser() -parser.add_argument("--train_data_path", type=str, required=True) -parser.add_argument("--submission_path", type=str, required=True) -parser.add_argument("--test_data_path", type=str, required=True) -parser.add_argument("--best_model_path", type=str, required=True) -# Optional -parser.add_argument("--backbone", type=str, default="resnet18") -parser.add_argument("--learning_rate", type=float, default=0.01) -args = parser.parse_args() - - -datamodule = ImageClassificationData.from_folders( - train_folder=args.train_data_path, - batch_size=8, -) - -model = ImageClassifier(datamodule.num_classes, backbone=args.backbone) -trainer = Trainer(fast_dev_run=True) -trainer.fit(model, datamodule=datamodule) -trainer.save_checkpoint(args.best_model_path) - -datamodule = ImageClassificationData.from_folders( - predict_folder=args.test_data_path, - batch_size=8, -) - -predictions = Trainer().predict(model, datamodule=datamodule) -submission_data = [ - {"filename": os.path.basename(p["metadata"]["filepath"]), "label": torch.argmax(p["preds"]).item()} - for batch in predictions - for p in batch -] -df = pd.DataFrame(submission_data) -df.to_csv(args.submission_path, index=False) diff --git a/examples/app/hpo/requirements.txt b/examples/app/hpo/requirements.txt deleted file mode 100644 index bd85880da2237..0000000000000 --- a/examples/app/hpo/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -optuna -lightning-flash[image,serve] == 0.7.0 -hiplot diff --git a/examples/app/hpo/utils.py b/examples/app/hpo/utils.py deleted file mode 100644 index a07ae73f8fd3e..0000000000000 --- a/examples/app/hpo/utils.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import os.path -import tarfile -import zipfile - -import requests - - -def download_data(url: str, path: str = "data/", verbose: bool = False) -> None: - """Download file with progressbar. - - # Code taken from: https://gist.github.com/ruxi/5d6803c116ec1130d484a4ab8c00c603 - # __author__ = "github.com/ruxi" - # __license__ = "MIT" - - Usage: - download_file('http://web4host.net/5MB.zip') - - """ - if url == "NEED_TO_BE_CREATED": - raise NotImplementedError - - if not os.path.exists(path): - os.makedirs(path) - local_filename = os.path.join(path, url.split("/")[-1]) - r = requests.get(url, stream=True, verify=False) - file_size = int(r.headers["Content-Length"]) if "Content-Length" in r.headers else 0 - chunk_size = 1024 - num_bars = int(file_size / chunk_size) - if verbose: - print({"file_size": file_size}) - print({"num_bars": num_bars}) - - if not os.path.exists(local_filename): - with open(local_filename, "wb") as fp: - for chunk in r.iter_content(chunk_size=chunk_size): - fp.write(chunk) # type: ignore - - def extract_tarfile(file_path: str, extract_path: str, mode: str): - if os.path.exists(file_path): - with tarfile.open(file_path, mode=mode) as tar_ref: - for member in tar_ref.getmembers(): - try: - tar_ref.extract(member, path=extract_path, set_attrs=False) - except PermissionError: - raise PermissionError(f"Could not extract tar file {file_path}") - - if ".zip" in local_filename: - if os.path.exists(local_filename): - with zipfile.ZipFile(local_filename, "r") as zip_ref: - zip_ref.extractall(path) # noqa: S202 - elif local_filename.endswith(".tar.gz") or local_filename.endswith(".tgz"): - extract_tarfile(local_filename, path, "r:gz") - elif local_filename.endswith(".tar.bz2") or local_filename.endswith(".tbz"): - extract_tarfile(local_filename, path, "r:bz2") diff --git a/examples/app/installation_commands/app.py b/examples/app/installation_commands/app.py deleted file mode 100644 index 526fcfef64413..0000000000000 --- a/examples/app/installation_commands/app.py +++ /dev/null @@ -1,31 +0,0 @@ -# EXAMPLE COMPONENT: RUN A SCRIPT -# app.py -# !echo "I am installing a dependency not declared in a requirements file" -# !pip install lmdb -import lmdb -from lightning.app import CloudCompute, LightningApp, LightningFlow, LightningWork - - -class YourComponent(LightningWork): - def run(self): - print(lmdb.version()) - print("lmdb successfully installed") - print("Accessing a module in a Work or Flow body works!") - - -class RootFlow(LightningFlow): - def __init__(self, work): - super().__init__() - self.work = work - - def run(self): - self.work.run() - - -print(f"Accessing an object in main code body works!: version = {lmdb.version()}") - - -# run on a cloud machine -compute = CloudCompute("cpu") -worker = YourComponent(cloud_compute=compute) -app = LightningApp(RootFlow(worker)) diff --git a/examples/app/interruptible/app.py b/examples/app/interruptible/app.py deleted file mode 100644 index a44fcf4dca3ed..0000000000000 --- a/examples/app/interruptible/app.py +++ /dev/null @@ -1,32 +0,0 @@ -from time import sleep - -from lightning.app import CloudCompute, LightningApp, LightningFlow, LightningWork - - -class Work(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.counter = 0 - - def run(self): - while True: - print(self.counter) - self.counter += 1 - sleep(1) - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.w = Work( - cloud_compute=CloudCompute("gpu", interruptible=True), - start_with_flow=False, - parallel=True, - ) - - def run(self): - self.w.run() - print(self.w.counter) - - -app = LightningApp(Flow()) diff --git a/examples/app/justpy/app.py b/examples/app/justpy/app.py deleted file mode 100644 index a4c9abc4cda1d..0000000000000 --- a/examples/app/justpy/app.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Callable - -from lightning import LightningApp, LightningFlow -from lightning.app.frontend import JustPyFrontend - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.counter = 0 - - def run(self): - print(self.counter) - - def configure_layout(self): - return JustPyFrontend(render_fn=render_fn) - - -def render_fn(get_state: Callable) -> Callable: - import justpy as jp - - def webpage(): - wp = jp.QuasarPage(dark=True) - d = jp.Div(classes="q-pa-md q-gutter-sm", a=wp) - container = jp.QBtn(color="primary", text="Counter: 0") - - async def click(*_): - state = get_state() - state.counter += 1 - container.text = f"Counter: {state.counter}" - - button = jp.QBtn(color="primary", text="Click Me!", click=click) - - d.add(button) - d.add(container) - - return wp - - return webpage - - -app = LightningApp(Flow()) diff --git a/examples/app/justpy/requirements.txt b/examples/app/justpy/requirements.txt deleted file mode 100644 index 5f69409a4e4bb..0000000000000 --- a/examples/app/justpy/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -justpy diff --git a/examples/app/layout/app.py b/examples/app/layout/app.py deleted file mode 100644 index c57ab2eff78e7..0000000000000 --- a/examples/app/layout/app.py +++ /dev/null @@ -1,101 +0,0 @@ -"""An example showcasing how `configure_layout` can be used to nest user interfaces of different flows. - -Run the app: - -lightning run app examples/layout/demo.py - -This starts one server for each flow that returns a UI. Access the UI at the link printed in the terminal. - -""" - -import os -from time import sleep - -from lightning.app import LightningApp, LightningFlow -from lightning.app.frontend import StaticWebFrontend, StreamlitFrontend - - -class C11(LightningFlow): - def __init__(self): - super().__init__() - self.message = "Hello Streamlit!" - - def run(self): - pass - - def configure_layout(self): - return StreamlitFrontend(render_fn=render_c11) - - -def render_c11(state): - import streamlit as st - - st.write(state.message) - - -class C21(LightningFlow): - def __init__(self): - super().__init__() - - def run(self): - pass - - def configure_layout(self): - return StaticWebFrontend(os.path.join(os.path.dirname(__file__), "ui1")) - - -class C22(LightningFlow): - def __init__(self): - super().__init__() - - def run(self): - pass - - def configure_layout(self): - return StaticWebFrontend(os.path.join(os.path.dirname(__file__), "ui2")) - - -class C1(LightningFlow): - def __init__(self): - super().__init__() - self.c11 = C11() - - def run(self): - pass - - -class C2(LightningFlow): - def __init__(self): - super().__init__() - self.c21 = C21() - self.c22 = C22() - - def run(self): - pass - - def configure_layout(self): - return [ - {"name": "one", "content": self.c21}, - {"name": "two", "content": self.c22}, - ] - - -class Root(LightningFlow): - def __init__(self): - super().__init__() - self.c1 = C1() - self.c2 = C2() - - def run(self): - sleep(10) - self.stop("Layout End") - - def configure_layout(self): - return [ - {"name": "one", "content": self.c1.c11}, - {"name": "two", "content": self.c2}, - {"name": "three", "content": "https://lightning.ai"}, - ] - - -app = LightningApp(Root()) diff --git a/examples/app/layout/requirements.txt b/examples/app/layout/requirements.txt deleted file mode 100644 index 12a4706528df6..0000000000000 --- a/examples/app/layout/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -streamlit diff --git a/examples/app/layout/ui1/index.html b/examples/app/layout/ui1/index.html deleted file mode 100644 index 11668dee1b911..0000000000000 --- a/examples/app/layout/ui1/index.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - - One - - - One - - diff --git a/examples/app/layout/ui2/index.html b/examples/app/layout/ui2/index.html deleted file mode 100644 index 7398be1f7630d..0000000000000 --- a/examples/app/layout/ui2/index.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - - Two - - - Two - - diff --git a/examples/app/mount/app.py b/examples/app/mount/app.py deleted file mode 100644 index b7e7c4df4746e..0000000000000 --- a/examples/app/mount/app.py +++ /dev/null @@ -1,34 +0,0 @@ -import os - -from lightning.app import CloudCompute, LightningApp, LightningFlow, LightningWork -from lightning.app.storage import Mount - - -class Work(LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def run(self): - files = os.listdir("/content/esRedditJson/") - for file in files: - print(file) - assert "esRedditJson1" in files - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.work_1 = Work( - cloud_compute=CloudCompute( - mounts=Mount( - source="s3://ryft-public-sample-data/esRedditJson/", - mount_path="/content/esRedditJson/", - ), - ) - ) - - def run(self): - self.work_1.run() - - -app = LightningApp(Flow()) diff --git a/examples/app/multi_node/README.md b/examples/app/multi_node/README.md deleted file mode 100644 index aef152444f4a4..0000000000000 --- a/examples/app/multi_node/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Lightning & Multi Node Training - -Lightning supports makes multi-node training simple by providing a simple interface to orchestrate compute and data. - -## Multi Node with raw PyTorch - -You can run the multi-node raw PyTorch by running the following commands. - -Here is an example where you spawn your processes yourself. - -```bash -lightning run app train_pytorch.py -``` - -or you can use the built-in component for it. - -```bash -lightning run app train_pytorch_spawn.py -``` - -## Multi Node with raw PyTorch + Fabric - -You can run the multi-node raw PyTorch and Fabric by running the following commands. - -```bash -lightning run app train_fabric.py -``` - -Using Fabric, you retain control over your loops while accessing in a minimal way all Lightning distributed strategies. - -## Multi Node with Lightning Trainer - -Lightning supports running Lightning Trainer from a script or within a Lightning Work. - -You can either run a script directly - -```bash -lightning run app train_pl_script.py -``` - -or run your code within as a work. - -```bash -lightning run app train_pl.py -``` - -## Multi Node with any frameworks - -```bash -lightning run app train_any.py -``` diff --git a/examples/app/multi_node/pl_boring_script.py b/examples/app/multi_node/pl_boring_script.py deleted file mode 100644 index f14809354f405..0000000000000 --- a/examples/app/multi_node/pl_boring_script.py +++ /dev/null @@ -1,7 +0,0 @@ -from lightning.pytorch import Trainer -from lightning.pytorch.demos.boring_classes import BoringModel - -if __name__ == "__main__": - model = BoringModel() - trainer = Trainer(max_epochs=1) - trainer.fit(model) diff --git a/examples/app/multi_node/requirements.txt b/examples/app/multi_node/requirements.txt deleted file mode 100644 index 12c6d5d5eac2a..0000000000000 --- a/examples/app/multi_node/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -torch diff --git a/examples/app/multi_node/train_any.py b/examples/app/multi_node/train_any.py deleted file mode 100644 index b3c89ad534f43..0000000000000 --- a/examples/app/multi_node/train_any.py +++ /dev/null @@ -1,22 +0,0 @@ -from lightning.app import CloudCompute, LightningApp, LightningWork -from lightning.app.components import MultiNode - - -class AnyDistributedComponent(LightningWork): - def run( - self, - main_address: str, - main_port: int, - num_nodes: int, - node_rank: int, - ): - print(f"ADD YOUR DISTRIBUTED CODE: {main_address} {main_port} {num_nodes} {node_rank}.") - - -app = LightningApp( - MultiNode( - AnyDistributedComponent, - num_nodes=2, - cloud_compute=CloudCompute("gpu"), - ) -) diff --git a/examples/app/multi_node/train_fabric.py b/examples/app/multi_node/train_fabric.py deleted file mode 100644 index 2379c491f89aa..0000000000000 --- a/examples/app/multi_node/train_fabric.py +++ /dev/null @@ -1,40 +0,0 @@ -import torch -from lightning.app import CloudCompute, LightningApp, LightningWork -from lightning.app.components import FabricMultiNode -from lightning.fabric import Fabric - - -class FabricPyTorchDistributed(LightningWork): - def run(self): - # 1. Prepare the model - model = torch.nn.Sequential( - torch.nn.Linear(1, 1), - torch.nn.ReLU(), - torch.nn.Linear(1, 1), - ) - - # 2. Create Fabric. - fabric = Fabric(strategy="ddp", precision="16-mixed") - model, optimizer = fabric.setup(model, torch.optim.SGD(model.parameters(), lr=0.01)) - criterion = torch.nn.MSELoss() - - # 3. Train the model for 1000 steps. - for step in range(1000): - model.zero_grad() - x = torch.tensor([0.8]).to(fabric.device) - target = torch.tensor([1.0]).to(fabric.device) - output = model(x) - loss = criterion(output, target) - print(f"global_rank: {fabric.global_rank} step: {step} loss: {loss}") - fabric.backward(loss) - optimizer.step() - - -# 8 GPUs: (2 nodes of 4 x v100) -app = LightningApp( - FabricMultiNode( - FabricPyTorchDistributed, - cloud_compute=CloudCompute("gpu-fast-multi"), # 4 x V100 - num_nodes=2, - ) -) diff --git a/examples/app/multi_node/train_lt.py b/examples/app/multi_node/train_lt.py deleted file mode 100644 index 23a2863e757c7..0000000000000 --- a/examples/app/multi_node/train_lt.py +++ /dev/null @@ -1,21 +0,0 @@ -# app.py -from lightning.app import CloudCompute, LightningApp, LightningWork -from lightning.app.components import LightningTrainerMultiNode -from lightning.pytorch import Trainer -from lightning.pytorch.demos.boring_classes import BoringModel - - -class LightningTrainerDistributed(LightningWork): - def run(self): - model = BoringModel() - trainer = Trainer(max_epochs=10, strategy="ddp") - trainer.fit(model) - - -# 8 GPUs: (2 nodes of 4 x v100) -component = LightningTrainerMultiNode( - LightningTrainerDistributed, - num_nodes=2, - cloud_compute=CloudCompute("gpu-fast-multi"), # 4 x v100 -) -app = LightningApp(component) diff --git a/examples/app/multi_node/train_lt_script.py b/examples/app/multi_node/train_lt_script.py deleted file mode 100644 index 7f89bc95e9b17..0000000000000 --- a/examples/app/multi_node/train_lt_script.py +++ /dev/null @@ -1,11 +0,0 @@ -from lightning.app import CloudCompute, LightningApp -from lightning.app.components import LightningTrainerScript - -# 8 GPUs: (2 nodes of 4 x v100) -app = LightningApp( - LightningTrainerScript( - "pl_boring_script.py", - num_nodes=2, - cloud_compute=CloudCompute("gpu-fast-multi"), # 4 x v100 - ), -) diff --git a/examples/app/multi_node/train_pytorch.py b/examples/app/multi_node/train_pytorch.py deleted file mode 100644 index a1c7fb8eac207..0000000000000 --- a/examples/app/multi_node/train_pytorch.py +++ /dev/null @@ -1,60 +0,0 @@ -# app.py -# ! pip install torch -import torch -from lightning.app import CloudCompute, LightningApp, LightningWork -from lightning.app.components import MultiNode -from torch.nn.parallel.distributed import DistributedDataParallel - - -def distributed_train(local_rank: int, main_address: str, main_port: int, num_nodes: int, node_rank: int, nprocs: int): - # 1. SET UP DISTRIBUTED ENVIRONMENT - global_rank = local_rank + node_rank * nprocs - world_size = num_nodes * nprocs - - if torch.distributed.is_available() and not torch.distributed.is_initialized(): - torch.distributed.init_process_group( - "nccl" if torch.cuda.is_available() else "gloo", - rank=global_rank, - world_size=world_size, - init_method=f"tcp://{main_address}:{main_port}", - ) - - # 2. PREPARE DISTRIBUTED MODEL - model = torch.nn.Linear(32, 2) - device = torch.device(f"cuda:{local_rank}") if torch.cuda.is_available() else torch.device("cpu") - model = DistributedDataParallel(model, device_ids=[local_rank] if torch.cuda.is_available() else None).to(device) - - # 3. SETUP LOSS AND OPTIMIZER - criterion = torch.nn.MSELoss() - optimizer = torch.optim.SGD(model.parameters(), lr=0.01) - - # 4.TRAIN THE MODEL FOR 50 STEPS - for step in range(50): - model.zero_grad() - x = torch.randn(64, 32).to(device) - output = model(x) - loss = criterion(output, torch.ones_like(output)) - print(f"global_rank: {global_rank} step: {step} loss: {loss}") - loss.backward() - optimizer.step() - - # 5. VERIFY ALL COPIES OF THE MODEL HAVE THE SAME WEIGTHS AT END OF TRAINING - weight = model.module.weight.clone() - torch.distributed.all_reduce(weight) - assert torch.equal(model.module.weight, weight / world_size) - - print("Multi Node Distributed Training Done!") - - -class PyTorchDistributed(LightningWork): - def run(self, main_address: str, main_port: int, num_nodes: int, node_rank: int): - nprocs = torch.cuda.device_count() if torch.cuda.is_available() else 1 - torch.multiprocessing.spawn( - distributed_train, args=(main_address, main_port, num_nodes, node_rank, nprocs), nprocs=nprocs - ) - - -# 8 GPUs: (2 nodes x 4 v 100) -compute = CloudCompute("gpu-fast-multi") # 4 x v100 -component = MultiNode(PyTorchDistributed, num_nodes=2, cloud_compute=compute) -app = LightningApp(component) diff --git a/examples/app/multi_node/train_pytorch_spawn.py b/examples/app/multi_node/train_pytorch_spawn.py deleted file mode 100644 index 8febfe5dcf696..0000000000000 --- a/examples/app/multi_node/train_pytorch_spawn.py +++ /dev/null @@ -1,51 +0,0 @@ -import torch -from lightning.app import CloudCompute, LightningApp, LightningWork -from lightning.app.components import PyTorchSpawnMultiNode -from torch.nn.parallel.distributed import DistributedDataParallel - - -class PyTorchDistributed(LightningWork): - def run( - self, - world_size: int, - node_rank: int, - global_rank: str, - local_rank: int, - ): - # 1. Prepare the model - model = torch.nn.Sequential( - torch.nn.Linear(1, 1), - torch.nn.ReLU(), - torch.nn.Linear(1, 1), - ) - - # 2. Setup distributed training - device = torch.device(f"cuda:{local_rank}") if torch.cuda.is_available() else torch.device("cpu") - model = DistributedDataParallel( - model.to(device), device_ids=[local_rank] if torch.cuda.is_available() else None - ) - - # 3. Prepare loss and optimizer - criterion = torch.nn.MSELoss() - optimizer = torch.optim.SGD(model.parameters(), lr=0.01) - - # 4. Train the model for 1000 steps. - for step in range(1000): - model.zero_grad() - x = torch.tensor([0.8]).to(device) - target = torch.tensor([1.0]).to(device) - output = model(x) - loss = criterion(output, target) - print(f"global_rank: {global_rank} step: {step} loss: {loss}") - loss.backward() - optimizer.step() - - -# 8 GPUs: (2 nodes x 4 v 100) -app = LightningApp( - PyTorchSpawnMultiNode( - PyTorchDistributed, - num_nodes=2, - cloud_compute=CloudCompute("gpu-fast-multi"), # 4 x v100 - ) -) diff --git a/examples/app/payload/app.py b/examples/app/payload/app.py deleted file mode 100644 index c92f589b088cd..0000000000000 --- a/examples/app/payload/app.py +++ /dev/null @@ -1,31 +0,0 @@ -from lightning.app import LightningApp, LightningFlow, LightningWork -from lightning.app.storage import Payload - - -class SourceFileWriterWork(LightningWork): - def __init__(self): - super().__init__() - self.value = None - - def run(self): - self.value = Payload(42) - - -class DestinationWork(LightningWork): - def run(self, payload): - assert payload.value == 42 - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.src = SourceFileWriterWork() - self.dst = DestinationWork() - - def run(self): - self.src.run() - self.dst.run(self.src.value) - self.stop("Application End!") - - -app = LightningApp(RootFlow()) diff --git a/examples/app/pickle_or_not/app.py b/examples/app/pickle_or_not/app.py deleted file mode 100644 index aa7c3b01323da..0000000000000 --- a/examples/app/pickle_or_not/app.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging - -from lightning.app import LightningApp, LightningFlow, LightningWork - -logger = logging.getLogger(__name__) - - -class PickleChecker(LightningWork): - def run(self, pickle_image: bytes): - parsed = self.parse_image(pickle_image) - if parsed == b"it is a pickle": - return True - if parsed == b"it is not a pickle": - return False - raise Exception("Couldn't parse the image") - - @staticmethod - def parse_image(image_str: bytes): - return image_str - - -class Slack(LightningFlow): - def __init__(self): - super().__init__() - - @staticmethod - def send_message(message): - logger.info(f"Sending message: {message}") - - def run(self): - pass - - -class RootComponent(LightningFlow): - def __init__(self): - super().__init__() - self.pickle_checker = PickleChecker() - self.slack = Slack() - self.counter = 3 - - def run(self): - if self.counter > 0: - logger.info(f"Running the app {self.counter}") - image_str = b"it is not a pickle" - if self.pickle_checker.run(image_str): - self.slack.send_message("It's a pickle!") - else: - self.slack.send_message("It's not a pickle!") - self.counter -= 1 - else: - self.stop("Pickle or Not End") - - -app = LightningApp(RootComponent()) diff --git a/examples/app/pickle_or_not/requirements.txt b/examples/app/pickle_or_not/requirements.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/examples/app/server/app.py b/examples/app/server/app.py deleted file mode 100644 index 97030179ffa78..0000000000000 --- a/examples/app/server/app.py +++ /dev/null @@ -1,39 +0,0 @@ -# !pip install torchvision pydantic -import base64 -import io - -import torch -import torchvision -from lightning.app import CloudCompute, LightningApp -from lightning.app.components.serve import Image as InputImage -from lightning.app.components.serve import PythonServer -from PIL import Image -from pydantic import BaseModel - - -class PyTorchServer(PythonServer): - def setup(self): - self._model = torchvision.models.resnet18(pretrained=True) - self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self._model.to(self._device) - - def predict(self, request): - image = base64.b64decode(request.image.encode("utf-8")) - image = Image.open(io.BytesIO(image)) - transforms = torchvision.transforms.Compose([ - torchvision.transforms.Resize(224), - torchvision.transforms.ToTensor(), - torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ]) - image = transforms(image) - image = image.to(self._device) - prediction = self._model(image.unsqueeze(0)) - return {"prediction": prediction.argmax().item()} - - -class OutputData(BaseModel): - prediction: int - - -component = PyTorchServer(input_type=InputImage, output_type=OutputData, cloud_compute=CloudCompute("gpu")) -app = LightningApp(component) diff --git a/examples/app/server_with_auto_scaler/app.py b/examples/app/server_with_auto_scaler/app.py deleted file mode 100644 index 1320da6745fa6..0000000000000 --- a/examples/app/server_with_auto_scaler/app.py +++ /dev/null @@ -1,93 +0,0 @@ -# ! pip install torch torchvision -from typing import List - -import torch -import torchvision -from lightning.app import CloudCompute, LightningApp -from pydantic import BaseModel - - -class BatchRequestModel(BaseModel): - inputs: List[app.components.Image] - - -class BatchResponse(BaseModel): - outputs: List[app.components.Number] - - -class PyTorchServer(app.components.PythonServer): - def __init__(self, *args, **kwargs): - super().__init__( - input_type=BatchRequestModel, - output_type=BatchResponse, - *args, - **kwargs, - ) - - def setup(self): - if torch.cuda.is_available(): - self._device = torch.device("cuda:0") - elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): - self._device = torch.device("mps") - else: - self._device = torch.device("cpu") - self._model = torchvision.models.resnet18(pretrained=True).to(self._device) - - def predict(self, requests: BatchRequestModel): - transforms = torchvision.transforms.Compose([ - torchvision.transforms.Resize(224), - torchvision.transforms.ToTensor(), - torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ]) - images = [] - for request in requests.inputs: - image = app.components.serve.types.image.Image.deserialize(request.image) - image = transforms(image).unsqueeze(0) - images.append(image) - images = torch.cat(images) - images = images.to(self._device) - predictions = self._model(images) - results = predictions.argmax(1).cpu().numpy().tolist() - return BatchResponse(outputs=[{"prediction": pred} for pred in results]) - - -class MyAutoScaler(app.components.AutoScaler): - def scale(self, replicas: int, metrics: dict) -> int: - pending_requests = metrics["pending_requests"] - active_or_pending_works = replicas + metrics["pending_works"] - - if active_or_pending_works == 0: - return 1 if pending_requests > 0 else 0 - - pending_requests_per_running_or_pending_work = pending_requests / active_or_pending_works - - # scale out if the number of pending requests exceeds max batch size. - max_requests_per_work = self.max_batch_size - if pending_requests_per_running_or_pending_work >= max_requests_per_work: - return replicas + 1 - - # scale in if the number of pending requests is below 25% of max_requests_per_work - min_requests_per_work = max_requests_per_work * 0.25 - if pending_requests_per_running_or_pending_work < min_requests_per_work: - return replicas - 1 - - return replicas - - -app = LightningApp( - MyAutoScaler( - # work class and args - PyTorchServer, - cloud_compute=CloudCompute("gpu"), - # autoscaler specific args - min_replicas=1, - max_replicas=4, - scale_out_interval=10, - scale_in_interval=10, - endpoint="predict", - input_type=app.components.Image, - output_type=app.components.Number, - timeout_batching=1, - max_batch_size=8, - ) -) diff --git a/examples/app/template_streamlit_ui/app.py b/examples/app/template_streamlit_ui/app.py deleted file mode 100644 index 21a13036aa782..0000000000000 --- a/examples/app/template_streamlit_ui/app.py +++ /dev/null @@ -1,44 +0,0 @@ -from lightning.app import LightningApp, LightningFlow -from lightning.app.frontend import StreamlitFrontend -from lightning.app.utilities.state import AppState - - -class StreamlitUI(LightningFlow): - def __init__(self): - super().__init__() - self.message_to_print = "Hello World!" - self.should_print = False - - def configure_layout(self): - return StreamlitFrontend(render_fn=render_fn) - - -def render_fn(state: AppState): - import streamlit as st - - should_print = st.button("Should print to the terminal ?") - - if should_print: - state.should_print = not state.should_print - - st.write("Currently printing." if state.should_print else "Currently waiting to print.") - - -class HelloWorld(LightningFlow): - def __init__(self): - super().__init__() - self.counter = 0 - self.streamlit_ui = StreamlitUI() - - def run(self): - self.streamlit_ui.run() - if self.streamlit_ui.should_print: - print(f"{self.counter}: {self.streamlit_ui.message_to_print}") - self.counter += 1 - self.streamlit_ui.should_print = False - - def configure_layout(self): - return [{"name": "StreamLitUI", "content": self.streamlit_ui}] - - -app = LightningApp(HelloWorld()) diff --git a/examples/app/template_streamlit_ui/requirements.txt b/examples/app/template_streamlit_ui/requirements.txt deleted file mode 100644 index 12a4706528df6..0000000000000 --- a/examples/app/template_streamlit_ui/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -streamlit diff --git a/examples/app/v0/.gitignore b/examples/app/v0/.gitignore deleted file mode 100644 index 186149fa056fe..0000000000000 --- a/examples/app/v0/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.storage -.lightning diff --git a/examples/app/v0/README.md b/examples/app/v0/README.md deleted file mode 100644 index 516283ae9cedd..0000000000000 --- a/examples/app/v0/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# v0 app - -This app is a flow-only app with nothing fancy. -This is meant to present the basic functionalities of the lightning framework. - -## Starting it - -Local - -```bash -lightning run app app.py -``` - -Cloud - -```bash -lightning run app app.py --cloud -``` diff --git a/examples/app/v0/app.py b/examples/app/v0/app.py deleted file mode 100644 index d1cbb41c6dc10..0000000000000 --- a/examples/app/v0/app.py +++ /dev/null @@ -1,49 +0,0 @@ -# v0_app.py -import os -from datetime import datetime -from time import sleep - -from lightning.app import LightningApp, LightningFlow -from lightning.app.frontend import StaticWebFrontend - - -class Word(LightningFlow): - def __init__(self, letter): - super().__init__() - self.letter = letter - self.repeats = letter - - def run(self): - self.repeats += self.letter - - def configure_layout(self): - return StaticWebFrontend(os.path.join(os.path.dirname(__file__), f"ui/{self.letter}")) - - -class V0App(LightningFlow): - def __init__(self): - super().__init__() - self.aas = Word("a") - self.bbs = Word("b") - self.counter = 0 - - def run(self): - now = datetime.now() - now = now.strftime("%H:%M:%S") - log = {"time": now, "a": self.aas.repeats, "b": self.bbs.repeats} - print(log) - self.aas.run() - self.bbs.run() - - sleep(2.0) - self.counter += 1 - - def configure_layout(self): - tab1 = {"name": "Tab_1", "content": self.aas} - tab2 = {"name": "Tab_2", "content": self.bbs} - tab3 = {"name": "Tab_3", "content": "https://tensorboard.dev/experiment/8m1aX0gcQ7aEmH0J7kbBtg/#scalars"} - - return [tab1, tab2, tab3] - - -app = LightningApp(V0App(), log_level="debug") diff --git a/examples/app/v0/emulate_ui.py b/examples/app/v0/emulate_ui.py deleted file mode 100644 index 1d42c1cdf4c52..0000000000000 --- a/examples/app/v0/emulate_ui.py +++ /dev/null @@ -1,18 +0,0 @@ -from time import sleep - -import requests -from lightning.app.utilities.state import headers_for - -headers = headers_for({}) -headers["X-Lightning-Type"] = "DEFAULT" - -res = requests.get("http://127.0.0.1:7501/state", headers=headers) - - -res = requests.post("http://127.0.0.1:7501/state", json={"stage": "running"}, headers=headers) -print(res) - -sleep(10) - -res = requests.post("http://127.0.0.1:7501/state", json={"stage": "stopping"}, headers=headers) -print(res) diff --git a/examples/app/v0/requirements.txt b/examples/app/v0/requirements.txt deleted file mode 100644 index edfce786a4d18..0000000000000 --- a/examples/app/v0/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -py diff --git a/examples/app/v0/ui/a/index.html b/examples/app/v0/ui/a/index.html deleted file mode 100644 index 6ddb9a5a1323c..0000000000000 --- a/examples/app/v0/ui/a/index.html +++ /dev/null @@ -1 +0,0 @@ -
Hello from component A
diff --git a/examples/app/v0/ui/b/index.html b/examples/app/v0/ui/b/index.html deleted file mode 100644 index 3bfd9e24cb7f7..0000000000000 --- a/examples/app/v0/ui/b/index.html +++ /dev/null @@ -1 +0,0 @@ -
Hello from component B
diff --git a/examples/app/works_on_default_machine/app_v2.py b/examples/app/works_on_default_machine/app_v2.py deleted file mode 100644 index 191070041b866..0000000000000 --- a/examples/app/works_on_default_machine/app_v2.py +++ /dev/null @@ -1,52 +0,0 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from lightning import CloudCompute, LightningApp, LightningFlow, LightningWork -from uvicorn import run - - -class Work(LightningWork): - def __init__(self, **kwargs): - super().__init__(parallel=True, **kwargs) - - def run(self): - fastapi_service = FastAPI() - - fastapi_service.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - - @fastapi_service.get("/") - def get_root(): - return {"Hello Word!"} - - run(fastapi_service, host=self.host, port=self.port) - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # In the Cloud: All the works defined without passing explicitly a CloudCompute object - # are running on the default machine. - # This would apply to `work_a`, `work_b` and the dynamically created `work_d`. - - self.work_a = Work() - self.work_b = Work() - - self.work_c = Work(cloud_compute=CloudCompute(name="cpu-small")) - - def run(self): - if not hasattr(self, "work_d"): - self.work_d = Work() - - for work in self.works(): - work.run() - - def configure_layout(self): - return [{"name": w.name, "content": w} for i, w in enumerate(self.works())] - - -app = LightningApp(Flow(), log_level="debug") diff --git a/examples/app/works_on_default_machine/requirements.txt b/examples/app/works_on_default_machine/requirements.txt deleted file mode 100644 index 12a4706528df6..0000000000000 --- a/examples/app/works_on_default_machine/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -streamlit diff --git a/pyproject.toml b/pyproject.toml index c24f27828fdd6..4189de8e9790a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,14 +134,7 @@ max-complexity = 10 files = [ "src/lightning", ] -# This section is for folders with "-" as they are not valid python modules -exclude = [ - "src/lightning/app/cli/app-template", - "src/lightning/app/cli/component-template", - "src/lightning/app/cli/pl-app-template", - "src/lightning/app/cli/react-ui-template", - "src/lightning/app/launcher/utils.py", -] + install_types = "True" non_interactive = "True" disallow_untyped_defs = "True" @@ -156,100 +149,6 @@ disable_error_code = "attr-defined" # style choices warn_no_return = "False" -# Ignore mypy errors for these files -# TODO: the goal is for this to be empty -[[tool.mypy.overrides]] -# the list can be generated with: -# mypy --no-error-summary 2>&1 | tr ':' ' ' | awk '{print $1}' | sort | uniq | sed 's/\.py//g; s|src/||g; s|\/|\.|g' | xargs -I {} echo '"{}",' -module = [ - "lightning.app.api.http_methods", - "lightning.app.api.request_types", - "lightning.app.cli.cmd_install", - "lightning.app.cli.commands.app_commands", - "lightning.app.cli.commands.cd", - "lightning.app.cli.commands.cp", - "lightning.app.cli.commands.ls", - "lightning.app.cli.connect.app", - "lightning.app.components.database.client", - "lightning.app.components.database.server", - "lightning.app.components.database.utilities", - "lightning.app.components.multi_node.base", - "lightning.app.components.multi_node.fabric", - "lightning.app.components.multi_node.pytorch_spawn", - "lightning.app.components.multi_node.trainer", - "lightning.app.components.python.popen", - "lightning.app.components.python.tracer", - "lightning.app.components.serve.auto_scaler", - "lightning.app.components.serve.gradio_server", - "lightning.app.components.serve.python_server", - "lightning.app.components.serve.serve", - "lightning.app.components.serve.streamlit", - "lightning.app.components.serve.types.image", - "lightning.app.components.serve.types.type", - "lightning.app.components.training", - "lightning.app.frontend.panel.app_state_comm", - "lightning.app.frontend.panel.app_state_watcher", - "lightning.app.frontend.panel.panel_frontend", - "lightning.app.frontend.panel.panel_serve_render_fn", - "lightning.app.frontend.streamlit_base", - "lightning.app.frontend.stream_lit", - "lightning.app.frontend.utils", - "lightning.app.frontend.web", - "lightning.app.launcher.launcher", - "lightning.app.launcher.lightning_backend", - "lightning.app.launcher.lightning_hybrid_backend", - "lightning.app.pdb.pdb", - "lightning.app.runners.backends.backend", - "lightning.app.runners.backends.cloud", - "lightning.app.runners.backends.docker", - "lightning.app.runners.backends.mp_process", - "lightning.app.runners.cloud", - "lightning.app.runners.multiprocess", - "lightning.app.runners.runtime", - "lightning.app.source_code.copytree", - "lightning.app.source_code.hashing", - "lightning.app.source_code.local", - "lightning.app.source_code.tar", - "lightning.app.source_code.uploader", - "lightning.app.storage.copier", - "lightning.app.storage.drive", - "lightning.app.storage.filesystem", - "lightning.app.storage.orchestrator", - "lightning.app.storage.path", - "lightning.app.storage.payload", - "lightning.app.structures.dict", - "lightning.app.structures.list", - "lightning.app.testing.helpers", - "lightning.app.testing.testing", - "lightning.app.utilities.app_helpers", - "lightning.app.utilities.app_logs", - "lightning.app.utilities.cli_helpers", - "lightning.app.utilities.cloud", - "lightning.app.utilities.commands.base", - "lightning.app.utilities.component", - "lightning.app.utilities.enum", - "lightning.app.utilities.exceptions", - "lightning.app.utilities.git", - "lightning.app.utilities.imports", - "lightning.app.utilities.introspection", - "lightning.app.utilities.layout", - "lightning.app.utilities.load_app", - "lightning.app.utilities.log_helpers", - "lightning.app.utilities.login", - "lightning.app.utilities.name_generator", - "lightning.app.utilities.network", - "lightning.app.utilities.openapi", - "lightning.app.utilities.packaging.cloud_compute", - "lightning.app.utilities.packaging.lightning_utils", - "lightning.app.utilities.proxies", - "lightning.app.utilities.scheduler", - "lightning.app.utilities.state", - "lightning.app.utilities.tracer", - "lightning.app.utilities.tree", - "lightning.store.utils", -] -ignore_errors = "True" - [tool.coverage.report] exclude_lines = [ diff --git a/requirements.txt b/requirements.txt index bcb63693fbbe3..4910c7fbe7fc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ # the default package dependencies --r ./requirements/app/app.txt -r ./requirements/fabric/base.txt -r ./requirements/pytorch/base.txt diff --git a/requirements/app/app.txt b/requirements/app/app.txt deleted file mode 100644 index 25c9bb893fe60..0000000000000 --- a/requirements/app/app.txt +++ /dev/null @@ -1,31 +0,0 @@ -lightning-cloud == 0.5.70 # Must be pinned to ensure compatibility -packaging -typing-extensions >=4.4.0, <4.10.0 -deepdiff >=5.7.0, <6.6.0 -fsspec[http] >=2022.5.0, <2023.11.0 -croniter >=1.3.0, <1.5.0 # strict; TODO: for now until we find something more robust. -traitlets >=5.3.0, <5.12.0 -arrow >=1.2.0, <1.3.0 -lightning-utilities >=0.10.0, <0.12.0 -beautifulsoup4 >=4.8.0, <4.13.0 -inquirer >=2.10.0, <3.2.0 -psutil <5.9.6 -click <8.2 -python-multipart >=0.0.5, <=0.0.6 -backoff >=2.2.1, <2.3.0 - -fastapi >=0.92.0, <0.104.0 -starlette # https://fastapi.tiangolo.com/deployment/versions/#about-starlette -pydantic >=1.7.4 # https://fastapi.tiangolo.com/deployment/versions/#about-pydantic - -dateutils <0.8.0 -Jinja2 <3.2.0 -PyYAML <=6.0.1 -requests <2.32.0 -rich >=12.3.0, <13.6.0 -urllib3 <2.0.0 -uvicorn <0.24.0 -websocket-client <1.7.0 -websockets <11.1.0 -numpy >=1.17.2, <2.0 -msgpack diff --git a/requirements/app/cloud.txt b/requirements/app/cloud.txt deleted file mode 100644 index ad5d2d583d17f..0000000000000 --- a/requirements/app/cloud.txt +++ /dev/null @@ -1,4 +0,0 @@ -redis >=4.0.1, <5.1.0 -docker >=5.0.0, <6.1.4 -s3fs >=2022.5.0, <2023.6.1 -# setuptools==59.5.0 diff --git a/requirements/app/components.txt b/requirements/app/components.txt deleted file mode 100644 index 78509b6b0269e..0000000000000 --- a/requirements/app/components.txt +++ /dev/null @@ -1,5 +0,0 @@ -# deps required by components in the lightning app repository (src/lightning/app/components) -lightning_api_access >=0.0.3 # serve -aiohttp >=3.8.0, <3.9.0 # auto_scaler -lightning-fabric >=1.9.0 # multinode -pytorch-lightning >=1.9.0 # multinode diff --git a/requirements/app/docs.txt b/requirements/app/docs.txt deleted file mode 100644 index f2db5000b9113..0000000000000 --- a/requirements/app/docs.txt +++ /dev/null @@ -1 +0,0 @@ --r ../docs.txt diff --git a/requirements/app/test.txt b/requirements/app/test.txt deleted file mode 100644 index fd9629649c89b..0000000000000 --- a/requirements/app/test.txt +++ /dev/null @@ -1,18 +0,0 @@ -coverage ==7.3.1 -pytest ==7.4.0 -pytest-timeout ==2.1.0 -pytest-cov ==4.1.0 -pytest-doctestplus ==1.0.0 -pytest-asyncio ==0.21.1 -# pytest-random-order ==1.1.0 -pytest-rerunfailures ==12.0 -pytest-xdist ==3.3.1 - -playwright ==1.38.0 -httpx ==0.25.0 -trio <0.22.0 # strict https://github.com/python-trio/trio/pull/2213 -pympler -psutil <5.10.0 -setuptools <68.3.0 -requests-mock ==1.11.0 -pandas diff --git a/requirements/app/ui.txt b/requirements/app/ui.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/setup.py b/setup.py index 698eaa5abe71e..bfc329bb8fe88 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ There are considered three main scenarios for installing this project: -1. Using PyPI registry when you can install `pytorch-lightning`, `lightning-app`, etc. or `lightning` for all. +1. Using PyPI registry when you can install `pytorch-lightning`, etc. or `lightning` for all. 2. Installation from source code after cloning repository. In such case we recommend to use command `pip install .` or `pip install -e .` for development version @@ -26,12 +26,11 @@ - for `pytorch-lightning` use `export PACKAGE_NAME=pytorch ; pip install .` - for `lightning-fabric` use `export PACKAGE_NAME=fabric ; pip install .` - - for `lightning-app` use `export PACKAGE_NAME=app ; pip install .` 3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command `python setup.py sdist` or `python setup.py bdist_wheel` accordingly. In case you want to build just a particular package you want to set an environment variable: - `PACKAGE_NAME=lightning|pytorch|app|fabric python setup.py sdist|bdist_wheel` + `PACKAGE_NAME=lightning|pytorch|fabric python setup.py sdist|bdist_wheel` 4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps: a) determine which packages shall be released based on version increment in `__version__.py` and eventually @@ -57,7 +56,6 @@ _PACKAGE_MAPPING = { "lightning": "lightning", "pytorch": "pytorch_lightning", - "app": "lightning_app", "fabric": "lightning_fabric", } # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/src/app-ui-version.info b/src/app-ui-version.info deleted file mode 100644 index ae39fab35ff1f..0000000000000 --- a/src/app-ui-version.info +++ /dev/null @@ -1 +0,0 @@ -v0.0.0 diff --git a/src/lightning/__init__.py b/src/lightning/__init__.py index c191334d2c218..1b054ed6715f7 100644 --- a/src/lightning/__init__.py +++ b/src/lightning/__init__.py @@ -1,7 +1,6 @@ """Root package info.""" import logging -import sys # explicitly don't set root logger's propagation and leave this to subpackages to manage _logger = logging.getLogger(__name__) @@ -31,19 +30,3 @@ "Fabric", "__version__", ] - - -def _cli_entry_point() -> None: - from lightning_utilities.core.imports import ModuleAvailableCache, RequirementCache - - if not ( - ModuleAvailableCache("lightning.app") - if RequirementCache("lightning-utilities<0.10.0") - else RequirementCache(module="lightning.app") - ): - print("The `lightning` command requires additional dependencies: `pip install lightning[app]`") - sys.exit(1) - - from lightning.app.cli.lightning_cli import main - - main() diff --git a/src/lightning/__main__.py b/src/lightning/__main__.py deleted file mode 100644 index 57b27ab968c82..0000000000000 --- a/src/lightning/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from lightning.app.cli.lightning_cli import main - -if __name__ == "__main__": - main() diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 81eae48545180..4bc4dff23be50 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -45,12 +45,8 @@ def _prepare_extras() -> Dict[str, Any]: extras["fabric-dev"] = extras["fabric-all"] + extras["fabric-test"] extras["pytorch-all"] = extras["pytorch-extra"] + extras["pytorch-strategies"] + extras["pytorch-examples"] extras["pytorch-dev"] = extras["pytorch-all"] + extras["pytorch-test"] - extras["app-extra"] = extras["app-app"] + extras["app-cloud"] + extras["app-ui"] + extras["app-components"] - extras["app-all"] = extras["app-extra"] - extras["app-dev"] = extras["app-all"] + extras["app-test"] - extras["store-store"] = extras["app-app"] # todo: consider cutting/leaning this dependency - # merge per-project extras of the same category, e.g. `app-test` + `fabric-test` + # merge per-project extras of the same category for extra in list(extras): name = "-".join(extra.split("-")[1:]) extras[name] = extras.get(name, []) + extras[extra] @@ -74,17 +70,6 @@ def _setup_args() -> Dict[str, Any]: _PROJECT_ROOT, homepage=about.__homepage__, version=version.version ) - # TODO: remove this once lightning-ui package is ready as a dependency - ui_ver_file = os.path.join(_SOURCE_ROOT, "app-ui-version.info") - if os.path.isfile(ui_ver_file): - with open(ui_ver_file, encoding="utf-8") as fo: - ui_version = fo.readlines()[0].strip() - download_fe_version = {"version": ui_version} - else: - print(f"Missing file with FE version: {ui_ver_file}") - download_fe_version = {} - _ASSISTANT._download_frontend(os.path.join(_PACKAGE_ROOT, "app"), **download_fe_version) - # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip install_requires = _ASSISTANT.load_requirements( @@ -114,7 +99,6 @@ def _setup_args() -> Dict[str, Any]: "console_scripts": [ "fabric = lightning.fabric.cli:_main", "lightning = lightning.fabric.cli:_legacy_main", - "lightning_app = lightning:_cli_entry_point", ], }, "setup_requires": [], diff --git a/src/lightning/app/CHANGELOG.md b/src/lightning/app/CHANGELOG.md deleted file mode 100644 index d09c302b22067..0000000000000 --- a/src/lightning/app/CHANGELOG.md +++ /dev/null @@ -1,608 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - -## [2.2.0] - 2024-02-09 - -## Changed - -- Renames the `lightning` cli to `lightning_app` ([#19440](https://github.com/Lightning-AI/pytorch-lightning/pull/19440)) - - -## [2.1.4] - 2024-01-31 - -### Changed - -- Remove torch distributed for the Dataset Optimizer ([#19182](https://github.com/Lightning-AI/lightning/pull/19182)) - - -## [2.1.3] - 2023-12-21 - -### Changed - -- Lightning App: Use the batch get endpoint ([#19180](https://github.com/Lightning-AI/lightning/pull/19180)) -- Drop starsessions from App's requirements ([#18470](https://github.com/Lightning-AI/lightning/pull/18470)) -- Optimize loading time for chunks to be there ([#19109](https://github.com/Lightning-AI/lightning/pull/19109)) - - -## [2.1.2] - 2023-11-15 - -### Changed - -- Forced plugin server to use localhost ([#18976](https://github.com/Lightning-AI/lightning/pull/18976)) -- Enabled bundling additional files into app source ([#18980](https://github.com/Lightning-AI/lightning/pull/18980)) -- Limited rate of requests to http queue ([#18981](https://github.com/Lightning-AI/lightning/pull/18981)) - - -## [2.1.1] - 2023-11-06 - -### Added - -- Added flow `fail()` ([#18883](https://github.com/Lightning-AI/lightning/pull/18883)) - -### Fixed - -- Fixed failing lightning cli entry point ([#18821](https://github.com/Lightning-AI/lightning/pull/18821)) - - -## [2.1.0] - 2023-10-11 - -### Added - -- Allow customizing `gradio` components with lightning colors ([#17054](https://github.com/Lightning-AI/lightning/pull/17054)) - -### Changed - -- Changed `LocalSourceCodeDir` cache_location to not use home in some certain cases ([#17491](https://github.com/Lightning-AI/lightning/pull/17491)) - -### Removed - -- Remove cluster commands from the CLI ([#18151](https://github.com/Lightning-AI/lightning/pull/18151)) - - -## [2.0.9] - 2023-09-14 - -### Fixed - -- Replace LightningClient with import from lightning_cloud ([#18544](https://github.com/Lightning-AI/lightning/pull/18544)) - - -## [2.0.8] - 2023-08-29 - -## Changed - -- Change top folder ([#18212](https://github.com/Lightning-AI/lightning/pull/18212)) -- Remove `_handle_is_headless` calls in app run loop ([#18362](https://github.com/Lightning-AI/lightning/pull/18362)) - - -## [2.0.7] - 2023-08-14 - -### Changed - -- Removed the top-level import `lightning.pdb`; import `lightning.app.pdb` instead ([#18177](https://github.com/Lightning-AI/lightning/pull/18177)) -- Client retries forever ([#18065](https://github.com/Lightning-AI/lightning/pull/18065)) - -### Fixed - -- Fixed an issue that would prevent the user to set the multiprocessing start method after importing lightning ([#18177](https://github.com/Lightning-AI/lightning/pull/18177)) - - -## [2.0.6] - 2023-07-20 - -### Fixed - -- Fixed handling a `None` request in the file orchestration queue ([#18111](https://github.com/Lightning-AI/lightning/pull/18111)) - - -## [2.0.5] - 2023-07-07 - -### Added - -- plugin: store source app ([#17892](https://github.com/Lightning-AI/lightning/pull/17892)) -- added colocation identifier ([#16796](https://github.com/Lightning-AI/lightning/pull/16796)) -- Added exponential backoff to HTTPQueue put ([#18013](https://github.com/Lightning-AI/lightning/pull/18013)) -- Content for plugins ([#17243](https://github.com/Lightning-AI/lightning/pull/17243)) - -### Changed - -- Save a reference to created tasks, to avoid tasks disappearing ([#17946](https://github.com/Lightning-AI/lightning/pull/17946)) - - -## [2.0.4] - 2023-06-22 - -### Fixed - -- bumped several dependencies to address security vulnerabilities. - - -## [2.0.3] - 2023-06-07 - -- Added the property `LightningWork.public_ip` that exposes the public IP of the `LightningWork` instance ([#17742](https://github.com/Lightning-AI/lightning/pull/17742)) -- Add missing python-multipart dependency ([#17244](https://github.com/Lightning-AI/lightning/pull/17244)) - -### Changed - -- Made type hints public ([#17100](https://github.com/Lightning-AI/lightning/pull/17100)) - -### Fixed - -- Fixed `LightningWork.internal_ip` that was mistakenly exposing the public IP instead; now exposes the private/internal IP address ([#17742](https://github.com/Lightning-AI/lightning/pull/17742)) -- Fixed resolution of latest version in CLI ([#17351](https://github.com/Lightning-AI/lightning/pull/17351)) -- Fixed property raised instead of returned ([#17595](https://github.com/Lightning-AI/lightning/pull/17595)) -- Fixed get project ([#17617](https://github.com/Lightning-AI/lightning/pull/17617), [#17666](https://github.com/Lightning-AI/lightning/pull/17666)) - - -## [2.0.2] - 2023-04-24 - -### Fixed - -- Resolved Lightning App with remote storage ([#17426](https://github.com/Lightning-AI/lightning/pull/17426)) -- Fixed `AppState`, streamlit example ([#17452](https://github.com/Lightning-AI/lightning/pull/17452)) - - -## [2.0.1] - 2023-04-11 - -### Fixed - -- Fix frontend hosts when running with multi-process in the cloud ([#17324](https://github.com/Lightning-AI/lightning/pull/17324)) - - -## [2.0.0] - 2023-03-15 - -### Added - -- Added `--zip` option to the `lightning cp` command to copy content from the Cloud Platform Filesystem as a zipfile - -### Changed - -- Changed minimum supported version of `rich` from `10.14.0` to `12.13.0` ([#16798](https://github.com/Lightning-AI/lightning/pull/16798)) - -### Removed - -- Removed support for Python 3.7 ([#16579](https://github.com/Lightning-AI/lightning/pull/16579)) - - -## [1.9.4] - 2023-03-01 - -### Removed - -- Removed implicit ui testing with `testing.run_app_in_cloud` in favor of headless login and app selection ([#16741](https://github.com/Lightning-AI/lightning/pull/16741)) - - -## [1.9.3] - 2023-02-21 - -### Fixed - -- Fixed `lightning open` command and improved redirects ([#16794](https://github.com/Lightning-AI/lightning/pull/16794)) - - -## [1.9.2] - 2023-02-15 - -- Added Storage Commands ([#16740](https://github.com/Lightning-AI/lightning/pull/16740)) - * `rm`: Delete files from your Cloud Platform Filesystem -- Added `lightning connect data` to register data connection to private s3 buckets ([#16738](https://github.com/Lightning-AI/lightning/pull/16738)) - - -## [1.9.1] - 2023-02-10 - -### Added -- Added `lightning open` command ([#16482](https://github.com/Lightning-AI/lightning/pull/16482)) -- Added experimental support for interruptible GPU in the cloud ([#16399](https://github.com/Lightning-AI/lightning/pull/16399)) -- Added FileSystem abstraction to simply manipulation of files ([#16581](https://github.com/Lightning-AI/lightning/pull/16581)) -- Added Storage Commands ([#16606](https://github.com/Lightning-AI/lightning/pull/16606)) - * `ls`: List files from your Cloud Platform Filesystem - * `cd`: Change the current directory within your Cloud Platform filesystem (terminal session based) - * `pwd`: Return the current folder in your Cloud Platform Filesystem - * `cp`: Copy files between your Cloud Platform Filesystem and local filesystem -- Prevent to `cd` into non existent folders ([#16645](https://github.com/Lightning-AI/lightning/pull/16645)) -- Enabled `cp` (upload) at project level ([#16631](https://github.com/Lightning-AI/lightning/pull/16631)) -- Enabled `ls` and `cp` (download) at project level ([#16622](https://github.com/Lightning-AI/lightning/pull/16622)) -- Added `lightning connect data` to register data connection to s3 buckets ([#16670](https://github.com/Lightning-AI/lightning/pull/16670)) -- Added support for running with multiprocessing in the cloud ([#16624](https://github.com/Lightning-AI/lightning/pull/16624)) -- Initial plugin server ([#16523](https://github.com/Lightning-AI/lightning/pull/16523)) -- Connect and Disconnect node ([#16700](https://github.com/Lightning-AI/lightning/pull/16700)) - -### Changed - -- Changed the default `LightningClient(retry=False)` to `retry=True` ([#16382](https://github.com/Lightning-AI/lightning/pull/16382)) -- Add support for async predict method in PythonServer and remove torch context ([#16453](https://github.com/Lightning-AI/lightning/pull/16453)) -- Renamed `lightning.app.components.LiteMultiNode` to `lightning.app.components.FabricMultiNode` ([#16505](https://github.com/Lightning-AI/lightning/pull/16505)) -- Changed the command `lightning connect` to `lightning connect app` for consistency ([#16670](https://github.com/Lightning-AI/lightning/pull/16670)) -- Refactor cloud dispatch and update to new API ([#16456](https://github.com/Lightning-AI/lightning/pull/16456)) -- Updated app URLs to the latest format ([#16568](https://github.com/Lightning-AI/lightning/pull/16568)) - -### Fixed - -- Fixed a deadlock causing apps not to exit properly when running locally ([#16623](https://github.com/Lightning-AI/lightning/pull/16623)) -- Fixed the Drive root_folder not parsed properly ([#16454](https://github.com/Lightning-AI/lightning/pull/16454)) -- Fixed malformed path when downloading files using `lightning cp` ([#16626](https://github.com/Lightning-AI/lightning/pull/16626)) -- Fixed app name in URL ([#16575](https://github.com/Lightning-AI/lightning/pull/16575)) - - -## [1.9.0] - 2023-01-17 - -### Added - -- Added a possibility to set up basic authentication for Lightning apps ([#16105](https://github.com/Lightning-AI/lightning/pull/16105)) - -### Changed - -- The LoadBalancer now uses internal ip + port instead of URL exposed ([#16119](https://github.com/Lightning-AI/lightning/pull/16119)) -- Added support for logging in different trainer stages with `DeviceStatsMonitor` ([#16002](https://github.com/Lightning-AI/lightning/pull/16002)) -- Changed `lightning.app.components.serve.gradio` to `lightning.app.components.serve.gradio_server` ([#16201](https://github.com/Lightning-AI/lightning/pull/16201)) -- Made cluster creation/deletion async by default ([#16185](https://github.com/Lightning-AI/lightning/pull/16185)) -- Expose `LightningFlow.stop` method to stop the flow similar to works ([##16378](https://github.com/Lightning-AI/lightning/pull/16378)) - -### Fixed - -- Fixed not being able to run multiple lightning apps locally due to port collision ([#15819](https://github.com/Lightning-AI/lightning/pull/15819)) -- Avoid `relpath` bug on Windows ([#16164](https://github.com/Lightning-AI/lightning/pull/16164)) -- Avoid using the deprecated `LooseVersion` ([#16162](https://github.com/Lightning-AI/lightning/pull/16162)) -- Porting fixes to autoscaler component ([#16249](https://github.com/Lightning-AI/lightning/pull/16249)) -- Fixed a bug where `lightning login` with env variables would not correctly save the credentials ([#16339](https://github.com/Lightning-AI/lightning/pull/16339)) - - -## [1.8.6] - 2022-12-21 - -### Added - -- Added partial support for fastapi `Request` annotation in `configure_api` handlers ([#16047](https://github.com/Lightning-AI/lightning/pull/16047)) -- Added a nicer UI with URL and examples for the autoscaler component ([#16063](https://github.com/Lightning-AI/lightning/pull/16063)) -- Enabled users to have more control over scaling out/in interval ([#16093](https://github.com/Lightning-AI/lightning/pull/16093)) -- Added more datatypes to serving component ([#16018](https://github.com/Lightning-AI/lightning/pull/16018)) -- Added `work.delete` method to delete the work ([#16103](https://github.com/Lightning-AI/lightning/pull/16103)) -- Added `display_name` property to LightningWork for the cloud ([#16095](https://github.com/Lightning-AI/lightning/pull/16095)) -- Added `ColdStartProxy` to the AutoScaler ([#16094](https://github.com/Lightning-AI/lightning/pull/16094)) -- Added status endpoint, enable `ready` ([#16075](https://github.com/Lightning-AI/lightning/pull/16075)) -- Implemented `ready` for components ([#16129](https://github.com/Lightning-AI/lightning/pull/16129)) - -### Changed - -- The default `start_method` for creating Work processes locally on MacOS is now 'spawn' (previously 'fork') ([#16089](https://github.com/Lightning-AI/lightning/pull/16089)) -- The utility `lightning.app.utilities.cloud.is_running_in_cloud` now returns `True` during loading of the app locally when running with `--cloud` ([#16045](https://github.com/Lightning-AI/lightning/pull/16045)) -- Updated Multinode Warning ([#16091](https://github.com/Lightning-AI/lightning/pull/16091)) -- Updated app testing ([#16000](https://github.com/Lightning-AI/lightning/pull/16000)) -- Changed overwrite to `True` ([#16009](https://github.com/Lightning-AI/lightning/pull/16009)) -- Simplified messaging in cloud dispatch ([#16160](https://github.com/Lightning-AI/lightning/pull/16160)) -- Added annotations endpoint ([#16159](https://github.com/Lightning-AI/lightning/pull/16159)) - -### Fixed - -- Fixed `PythonServer` messaging "Your app has started" ([#15989](https://github.com/Lightning-AI/lightning/pull/15989)) -- Fixed auto-batching to enable batching for requests coming even after batch interval but is in the queue ([#16110](https://github.com/Lightning-AI/lightning/pull/16110)) -- Fixed a bug where `AutoScaler` would fail with min_replica=0 ([#16092](https://github.com/Lightning-AI/lightning/pull/16092) -- Fixed a non-thread safe deepcopy in the scheduler ([#16114](https://github.com/Lightning-AI/lightning/pull/16114)) -- Fixed Http Queue sleeping for 1 sec by default if no delta were found ([#16114](https://github.com/Lightning-AI/lightning/pull/16114)) -- Fixed the endpoint info tab not showing up in `AutoScaler` UI ([#16128](https://github.com/Lightning-AI/lightning/pull/16128)) -- Fixed an issue where an exception would be raised in the logs when using a recent version of streamlit ([#16139](https://github.com/Lightning-AI/lightning/pull/16139)) -- Fixed e2e tests ([#16146](https://github.com/Lightning-AI/lightning/pull/16146)) - - -## [1.8.5] - 2022-12-15 - -### Added - -- Added `Lightning{Flow,Work}.lightningignores` attributes to programmatically ignore files before uploading to the cloud ([#15818](https://github.com/Lightning-AI/lightning/pull/15818)) -- Added a progress bar while connecting to an app through the CLI ([#16035](https://github.com/Lightning-AI/lightning/pull/16035)) -- Support running on multiple clusters ([#16016](https://github.com/Lightning-AI/lightning/pull/16016)) -- Added guards to cluster deletion from cli ([#16053](https://github.com/Lightning-AI/lightning/pull/16053)) - -### Changed - -- Cleanup cluster waiting ([#16054](https://github.com/Lightning-AI/lightning/pull/16054)) - -### Fixed - -- Fixed `DDPStrategy` import in app framework ([#16029](https://github.com/Lightning-AI/lightning/pull/16029)) -- Fixed `AutoScaler` raising an exception when non-default cloud compute is specified ([#15991](https://github.com/Lightning-AI/lightning/pull/15991)) -- Fixed and improvements of login flow ([#16052](https://github.com/Lightning-AI/lightning/pull/16052)) -- Fixed the debugger detection mechanism for lightning App in VSCode ([#16068](https://github.com/Lightning-AI/lightning/pull/16068)) -- Fixed bug where components that are re-instantiated several times failed to initialize if they were modifying `self.lightningignore` ([#16080](https://github.com/Lightning-AI/lightning/pull/16080)) -- Fixed a bug where apps that had previously been deleted could not be run again from the CLI ([#16082](https://github.com/Lightning-AI/lightning/pull/16082)) -- Fixed install/upgrade - removing single quote ([#16079](https://github.com/Lightning-AI/lightning/pull/16079)) - - -## [1.8.4] - 2022-12-08 - -### Added - -- Add `code_dir` argument to tracer run ([#15771](https://github.com/Lightning-AI/lightning/pull/15771)) -- Added the CLI command `lightning run model` to launch a `LightningLite` accelerated script ([#15506](https://github.com/Lightning-AI/lightning/pull/15506)) -- Added the CLI command `lightning delete app` to delete a lightning app on the cloud ([#15783](https://github.com/Lightning-AI/lightning/pull/15783)) -- Added a CloudMultiProcessBackend which enables running a child App from within the Flow in the cloud ([#15800](https://github.com/Lightning-AI/lightning/pull/15800)) -- Utility for pickling work object safely even from a child process ([#15836](https://github.com/Lightning-AI/lightning/pull/15836)) -- Added `AutoScaler` component ( - [#15769](https://github.com/Lightning-AI/lightning/pull/15769), - [#15971](https://github.com/Lightning-AI/lightning/pull/15971), - [#15966](https://github.com/Lightning-AI/lightning/pull/15966) -) -- Added the property `ready` of the LightningFlow to inform when the `Open App` should be visible ([#15921](https://github.com/Lightning-AI/lightning/pull/15921)) -- Added private work attributed `_start_method` to customize how to start the works ([#15923](https://github.com/Lightning-AI/lightning/pull/15923)) -- Added a `configure_layout` method to the `LightningWork` which can be used to control how the work is handled in the layout of a parent flow ([#15926](https://github.com/Lightning-AI/lightning/pull/15926)) -- Added the ability to run a Lightning App or Component directly from the Gallery using `lightning run app organization/name` ([#15941](https://github.com/Lightning-AI/lightning/pull/15941)) -- Added automatic conversion of list and dict of works and flows to structures ([#15961](https://github.com/Lightning-AI/lightning/pull/15961)) - -### Changed - -- The `MultiNode` components now warn the user when running with `num_nodes > 1` locally ([#15806](https://github.com/Lightning-AI/lightning/pull/15806)) -- Cluster creation and deletion now waits by default [#15458](https://github.com/Lightning-AI/lightning/pull/15458) -- Running an app without a UI locally no longer opens the browser ([#15875](https://github.com/Lightning-AI/lightning/pull/15875)) -- Show a message when `BuildConfig(requirements=[...])` is passed but a `requirements.txt` file is already present in the Work ([#15799](https://github.com/Lightning-AI/lightning/pull/15799)) -- Show a message when `BuildConfig(dockerfile="...")` is passed but a `Dockerfile` file is already present in the Work ([#15799](https://github.com/Lightning-AI/lightning/pull/15799)) -- Dropped name column from cluster list ([#15721](https://github.com/Lightning-AI/lightning/pull/15721)) -- Apps without UIs no longer activate the "Open App" button when running in the cloud ([#15875](https://github.com/Lightning-AI/lightning/pull/15875)) -- Wait for full file to be transferred in Path / Payload ([#15934](https://github.com/Lightning-AI/lightning/pull/15934)) - -### Removed - -- Removed the `SingleProcessRuntime` ([#15933](https://github.com/Lightning-AI/lightning/pull/15933)) - -### Fixed - -- Fixed SSH CLI command listing stopped components ([#15810](https://github.com/Lightning-AI/lightning/pull/15810)) -- Fixed bug when launching apps on multiple clusters ([#15484](https://github.com/Lightning-AI/lightning/pull/15484)) -- Fixed Sigterm Handler causing thread lock which caused KeyboardInterrupt to hang ([#15881](https://github.com/Lightning-AI/lightning/pull/15881)) -- Fixed MPS error for multinode component (defaults to cpu on mps devices now as distributed operations are not supported by pytorch on mps) ([#15748](https://github.com/Lightning-AI/lightning/pull/15748)) -- Fixed the work not stopped when successful when passed directly to the LightningApp ([#15801](https://github.com/Lightning-AI/lightning/pull/15801)) -- Fixed the PyTorch Inference locally on GPU ([#15813](https://github.com/Lightning-AI/lightning/pull/15813)) -- Fixed the `enable_spawn` method of the `WorkRunExecutor` ([#15812](https://github.com/Lightning-AI/lightning/pull/15812)) -- Fixed require/import decorator ([#15849](https://github.com/Lightning-AI/lightning/pull/15849)) -- Fixed a bug where using `L.app.structures` would cause multiple apps to be opened and fail with an error in the cloud ([#15911](https://github.com/Lightning-AI/lightning/pull/15911)) -- Fixed PythonServer generating noise on M1 ([#15949](https://github.com/Lightning-AI/lightning/pull/15949)) -- Fixed multiprocessing breakpoint ([#15950](https://github.com/Lightning-AI/lightning/pull/15950)) -- Fixed detection of a Lightning App running in debug mode ([#15951](https://github.com/Lightning-AI/lightning/pull/15951)) -- Fixed `ImportError` on Multinode if package not present ([#15963](https://github.com/Lightning-AI/lightning/pull/15963)) -- Fixed MultiNode Component to use separate cloud computes ([#15965](https://github.com/Lightning-AI/lightning/pull/15965)) -- Fixed Registration for CloudComputes of Works in `L.app.structures` ([#15964](https://github.com/Lightning-AI/lightning/pull/15964)) -- Fixed a bug where auto-upgrading to the latest lightning via the CLI could get stuck in a loop ([#15984](https://github.com/Lightning-AI/lightning/pull/15984)) - - -## [1.8.3] - 2022-11-22 - -### Changed - -- Deduplicate top level lightning CLI command groups ([#15761](https://github.com/Lightning-AI/lightning/pull/15761)) - * `lightning add ssh-key` CLI command has been transitioned to `lightning create ssh-key` - * `lightning remove ssh-key` CLI command has been transitioned to `lightning delete ssh-key` -- Set Torch inference mode for prediction ([#15719](https://github.com/Lightning-AI/lightning/pull/15719)) -- Improved `LightningTrainerScript` start-up time ([#15751](https://github.com/Lightning-AI/lightning/pull/15751)) -- Disable XSRF protection in `StreamlitFrontend` to support upload in localhost ([#15684](https://github.com/Lightning-AI/lightning/pull/15684)) - -### Fixed - -- Fixed debugging with VSCode IDE ([#15747](https://github.com/Lightning-AI/lightning/pull/15747)) -- Fixed setting property to the `LightningFlow` ([#15750](https://github.com/Lightning-AI/lightning/pull/15750)) -- Fixed the PyTorch Inference locally on GPU ([#15813](https://github.com/Lightning-AI/lightning/pull/15813)) - - -## [1.8.2] - 2022-11-17 - -### Added - -- Added title and description to ServeGradio ([#15639](https://github.com/Lightning-AI/lightning/pull/15639)) -- Added a friendly error message when attempting to run the default cloud compute with a custom base image configured ([#14929](https://github.com/Lightning-AI/lightning/pull/14929)) - -### Changed - -- Improved support for running apps when dependencies aren't installed ([#15711](https://github.com/Lightning-AI/lightning/pull/15711)) -- Changed the root directory of the app (which gets uploaded) to be the folder containing the app file, rather than any parent folder containing a `.lightning` file ([#15654](https://github.com/Lightning-AI/lightning/pull/15654)) -- Enabled MultiNode Components to support state broadcasting ([#15607](https://github.com/Lightning-AI/lightning/pull/15607)) -- Prevent artefactual "running from outside your current environment" error ([#15647](https://github.com/Lightning-AI/lightning/pull/15647)) -- Rename failed -> error in tables ([#15608](https://github.com/Lightning-AI/lightning/pull/15608)) - -### Fixed - -- Fixed race condition to over-write the frontend with app infos ([#15398](https://github.com/Lightning-AI/lightning/pull/15398)) -- Fixed bi-directional queues sending delta with Drive Component name changes ([#15642](https://github.com/Lightning-AI/lightning/pull/15642)) -- Fixed CloudRuntime works collection with structures and accelerated multi node startup time ([#15650](https://github.com/Lightning-AI/lightning/pull/15650)) -- Fixed catimage import ([#15712](https://github.com/Lightning-AI/lightning/pull/15712)) -- Parse all lines in app file looking for shebangs to run commands ([#15714](https://github.com/Lightning-AI/lightning/pull/15714)) - - -## [1.8.1] - 2022-11-10 - -### Added - -- Added the `start` method to the work ([#15523](https://github.com/Lightning-AI/lightning/pull/15523)) -- Added a `MultiNode` Component to run with distributed computation with any frameworks ([#15524](https://github.com/Lightning-AI/lightning/pull/15524)) -- Expose `RunWorkExecutor` to the work and provides default ones for the `MultiNode` Component ([#15561](https://github.com/Lightning-AI/lightning/pull/15561)) -- Added a `start_with_flow` flag to the `LightningWork` which can be disabled to prevent the work from starting at the same time as the flow ([#15591](https://github.com/Lightning-AI/lightning/pull/15591)) -- Added support for running Lightning App with VSCode IDE debugger ([#15590](https://github.com/Lightning-AI/lightning/pull/15590)) -- Added `bi-directional` delta updates between the flow and the works ([#15582](https://github.com/Lightning-AI/lightning/pull/15582)) -- Added `--setup` flag to `lightning run app` CLI command allowing for dependency installation via app comments ([#15577](https://github.com/Lightning-AI/lightning/pull/15577)) -- Auto-upgrade / detect environment mismatch from the CLI ([#15434](https://github.com/Lightning-AI/lightning/pull/15434)) -- Added Serve component ([#15609](https://github.com/Lightning-AI/lightning/pull/15609)) - - -### Changed - -- Changed the `flow.flows` to be recursive won't to align the behavior with the `flow.works` ([#15466](https://github.com/Lightning-AI/lightning/pull/15466)) -- The `params` argument in `TracerPythonScript.run` no longer prepends `--` automatically to parameters ([#15518](https://github.com/Lightning-AI/lightning/pull/15518)) -- Only check versions / env when not in the cloud ([#15504](https://github.com/Lightning-AI/lightning/pull/15504)) -- Periodically sync database to the drive ([#15441](https://github.com/Lightning-AI/lightning/pull/15441)) -- Slightly safer multi node ([#15538](https://github.com/Lightning-AI/lightning/pull/15538)) -- Reuse existing commands when running connect more than once ([#15471](https://github.com/Lightning-AI/lightning/pull/15471)) - -### Fixed - -- Fixed writing app name and id in connect.txt file for the command CLI ([#15443](https://github.com/Lightning-AI/lightning/pull/15443)) -- Fixed missing root flow among the flows of the app ([#15531](https://github.com/Lightning-AI/lightning/pull/15531)) -- Fixed bug with Multi Node Component and add some examples ([#15557](https://github.com/Lightning-AI/lightning/pull/15557)) -- Fixed a bug where payload would take a very long time locally ([#15557](https://github.com/Lightning-AI/lightning/pull/15557)) -- Fixed an issue with the `lightning` CLI taking a long time to error out when the cloud is not reachable ([#15412](https://github.com/Lightning-AI/lightning/pull/15412)) - - -## [1.8.0] - 2022-11-01 - -### Added - -- Added `load_state_dict` and `state_dict` hooks for `LightningFlow` components ([#14100](https://github.com/Lightning-AI/lightning/pull/14100)) -- Added a `--secret` option to CLI to allow binding secrets to app environment variables when running in the cloud ([#14612](https://github.com/Lightning-AI/lightning/pull/14612)) -- Added support for running the works without cloud compute in the default container ([#14819](https://github.com/Lightning-AI/lightning/pull/14819)) -- Added an HTTPQueue as an optional replacement for the default redis queue ([#14978](https://github.com/Lightning-AI/lightning/pull/14978) -- Added support for configuring flow cloud compute ([#14831](https://github.com/Lightning-AI/lightning/pull/14831)) -- Added support for adding descriptions to commands either through a docstring or the `DESCRIPTION` attribute ([#15193](https://github.com/Lightning-AI/lightning/pull/15193) -- Added a try / catch mechanism around request processing to avoid killing the flow ([#15187](https://github.com/Lightning-AI/lightning/pull/15187) -- Added an Database Component ([#14995](https://github.com/Lightning-AI/lightning/pull/14995) -- Added authentication to HTTP queue ([#15202](https://github.com/Lightning-AI/lightning/pull/15202)) -- Added support to pass a `LightningWork` to the `LightningApp` ([#15215](https://github.com/Lightning-AI/lightning/pull/15215) -- Added support getting CLI help for connected apps even if the app isn't running ([#15196](https://github.com/Lightning-AI/lightning/pull/15196) -- Added support for adding requirements to commands and installing them when missing when running an app command ([#15198](https://github.com/Lightning-AI/lightning/pull/15198) -- Added Lightning CLI Connection to be terminal session instead of global ([#15241](https://github.com/Lightning-AI/lightning/pull/15241) -- Added support for managing SSH-keys via CLI ([#15291](https://github.com/Lightning-AI/lightning/pull/15291)) -- Add a `JustPyFrontend` to ease UI creation with `https://github.com/justpy-org/justpy` ([#15002](https://github.com/Lightning-AI/lightning/pull/15002)) -- Added a layout endpoint to the Rest API and enable to disable pulling or pushing to the state ([#15367](https://github.com/Lightning-AI/lightning/pull/15367) -- Added support for functions for `configure_api` and `configure_commands` to be executed in the Rest API process ([#15098](https://github.com/Lightning-AI/lightning/pull/15098) -- Added support for accessing Lightning Apps via SSH ([#15310](https://github.com/Lightning-AI/lightning/pull/15310)) -- Added support to start lightning app on cloud without needing to install dependencies locally ([#15019](https://github.com/Lightning-AI/lightning/pull/15019) - -### Changed - -- Improved the show logs command to be standalone and reusable ([#15343](https://github.com/Lightning-AI/lightning/pull/15343) -- Removed the `--instance-types` option when creating clusters ([#15314](https://github.com/Lightning-AI/lightning/pull/15314)) - -### Fixed - -- Fixed an issue when using the CLI without arguments ([#14877](https://github.com/Lightning-AI/lightning/pull/14877)) -- Fixed a bug where the upload files endpoint would raise an error when running locally ([#14924](https://github.com/Lightning-AI/lightning/pull/14924)) -- Fixed BYOC cluster region selector -> hiding it from help since only us-east-1 has been tested and is recommended ([#15277]https://github.com/Lightning-AI/lightning/pull/15277) -- Fixed a bug when launching an app on multiple clusters ([#15226](https://github.com/Lightning-AI/lightning/pull/15226)) -- Fixed a bug with a default CloudCompute for Lightning flows ([#15371](https://github.com/Lightning-AI/lightning/pull/15371)) - -## [0.6.2] - 2022-09-21 - -### Changed - -- Improved Lightning App connect logic by disconnecting automatically ([#14532](https://github.com/Lightning-AI/lightning/pull/14532)) -- Improved the error message when the `LightningWork` is missing the `run` method ([#14759](https://github.com/Lightning-AI/lightning/pull/14759)) -- Improved the error message when the root `LightningFlow` passed to `LightningApp` is missing the `run` method ([#14760](https://github.com/Lightning-AI/lightning/pull/14760)) - -### Fixed - -- Fixed a bug where the uploaded command file wasn't properly parsed ([#14532](https://github.com/Lightning-AI/lightning/pull/14532)) -- Fixed an issue where custom property setters were not being used `LightningWork` class ([#14259](https://github.com/Lightning-AI/lightning/pull/14259)) -- Fixed an issue where some terminals would display broken icons in the PL app CLI ([#14226](https://github.com/Lightning-AI/lightning/pull/14226)) - - -## [0.6.1] - 2022-09-19 - -### Added - -- Add support to upload files to the Drive through an asynchronous `upload_file` endpoint ([#14703](https://github.com/Lightning-AI/lightning/pull/14703)) - -### Changed - -- Application storage prefix moved from `app_id` to `project_id/app_id` ([#14583](https://github.com/Lightning-AI/lightning/pull/14583)) -- LightningCloud client calls to use keyword arguments instead of positional arguments ([#14685](https://github.com/Lightning-AI/lightning/pull/14685)) - -### Fixed - -- Making `threadpool` non-default from LightningCloud client ([#14757](https://github.com/Lightning-AI/lightning/pull/14757)) -- Resolved a bug where the state change detection using DeepDiff won't work with Path, Drive objects ([#14465](https://github.com/Lightning-AI/lightning/pull/14465)) -- Resolved a bug where the wrong client was passed to collect cloud logs ([#14684](https://github.com/Lightning-AI/lightning/pull/14684)) -- Resolved the memory leak issue with the Lightning Cloud package and bumped the requirements to use the latest version ([#14697](https://github.com/Lightning-AI/lightning/pull/14697)) -- Fixing 5000 log line limitation for Lightning AI BYOC cluster logs ([#14458](https://github.com/Lightning-AI/lightning/pull/14458)) -- Fixed a bug where the uploaded command file wasn't properly parsed ([#14532](https://github.com/Lightning-AI/lightning/pull/14532)) -- Resolved `LightningApp(..., debug=True)` ([#14464](https://github.com/Lightning-AI/lightning/pull/14464)) - - -## [0.6.0] - 2022-09-08 - -### Added - -- Introduce lightning connect ([#14452](https://github.com/Lightning-AI/lightning/pull/14452)) -- Adds `PanelFrontend` to easily create complex UI in Python ([#13531](https://github.com/Lightning-AI/lightning/pull/13531)) -- Add support for `Lightning App Commands` through the `configure_commands` hook on the Lightning Flow and the `ClientCommand` ([#13602](https://github.com/Lightning-AI/lightning/pull/13602)) -- Add support for Lightning AI BYOC cluster management ([#13835](https://github.com/Lightning-AI/lightning/pull/13835)) -- Add support to see Lightning AI BYOC cluster logs ([#14334](https://github.com/Lightning-AI/lightning/pull/14334)) -- Add support to run Lightning apps on Lightning AI BYOC clusters ([#13894](https://github.com/Lightning-AI/lightning/pull/13894)) -- Add support for listing Lightning AI apps ([#13987](https://github.com/Lightning-AI/lightning/pull/13987)) -- Adds `LightningTrainerScript`. `LightningTrainerScript` orchestrates multi-node training in the cloud ([#13830](https://github.com/Lightning-AI/lightning/pull/13830)) -- Add support for printing application logs using CLI `lightning show logs [components]` ([#13634](https://github.com/Lightning-AI/lightning/pull/13634)) -- Add support for `Lightning API` through the `configure_api` hook on the Lightning Flow and the `Post`, `Get`, `Delete`, `Put` HttpMethods ([#13945](https://github.com/Lightning-AI/lightning/pull/13945)) -- Added a warning when `configure_layout` returns URLs configured with http instead of https ([#14233](https://github.com/Lightning-AI/lightning/pull/14233)) -- Add `--app_args` support from the CLI ([#13625](https://github.com/Lightning-AI/lightning/pull/13625)) - -### Changed - -- Default values and parameter names for Lightning AI BYOC cluster management ([#14132](https://github.com/Lightning-AI/lightning/pull/14132)) -- Run the flow only if the state has changed from the previous execution ([#14076](https://github.com/Lightning-AI/lightning/pull/14076)) -- Increased DeepDiff's verbose level to properly handle dict changes ([#13960](https://github.com/Lightning-AI/lightning/pull/13960)) -- Setup: added requirement freeze for next major version ([#14480](https://github.com/Lightning-AI/lightning/pull/14480)) - -### Fixed - -- Unification of app template: moved `app.py` to root dir for `lightning init app ` template ([#13853](https://github.com/Lightning-AI/lightning/pull/13853)) -- Fixed an issue with `lightning --version` command ([#14433](https://github.com/Lightning-AI/lightning/pull/14433)) -- Fixed imports of collections.abc for py3.10 ([#14345](https://github.com/Lightning-AI/lightning/pull/14345)) - -## [0.5.7] - 2022-08-22 - -### Changed - -- Release LAI docs as stable ([#14250](https://github.com/Lightning-AI/lightning/pull/14250)) -- Compatibility for Python 3.10 - -### Fixed - -- Pinning starsessions to 1.x ([#14333](https://github.com/Lightning-AI/lightning/pull/14333)) -- Parsed local package versions ([#13933](https://github.com/Lightning-AI/lightning/pull/13933)) - - -## [0.5.6] - 2022-08-16 - -### Fixed - -- Resolved a bug where the `install` command was not installing the latest version of an app/component by default ([#14181](https://github.com/Lightning-AI/lightning/pull/14181)) - - -- Fixed the `examples/app_dag` example ([#14359](https://github.com/Lightning-AI/lightning/pull/14359)) - - -## [0.5.5] - 2022-08-9 - -### Deprecated - -- Deprecate sheety API ([#14004](https://github.com/Lightning-AI/lightning/pull/14004)) - -### Fixed - -- Resolved a bug where the work statuses will grow quickly and be duplicated ([#13970](https://github.com/Lightning-AI/lightning/pull/13970)) -- Resolved a bug about a race condition when sending the work state through the caller_queue ([#14074](https://github.com/Lightning-AI/lightning/pull/14074)) -- Fixed Start Lightning App on Cloud if Repo Begins With Name "Lightning" ([#14025](https://github.com/Lightning-AI/lightning/pull/14025)) - - -## [0.5.4] - 2022-08-01 - -### Changed - -- Wrapped imports for traceability ([#13924](https://github.com/Lightning-AI/lightning/pull/13924)) -- Set version as today ([#13906](https://github.com/Lightning-AI/lightning/pull/13906)) - -### Fixed - -- Included app templates to the lightning and app packages ([#13731](https://github.com/Lightning-AI/lightning/pull/13731)) -- Added UI for install all ([#13732](https://github.com/Lightning-AI/lightning/pull/13732)) -- Fixed build meta pkg flow ([#13926](https://github.com/Lightning-AI/lightning/pull/13926)) - -## [0.5.3] - 2022-07-25 - -### Changed - -- Pruned requirements duplicity ([#13739](https://github.com/Lightning-AI/lightning/pull/13739)) - -### Fixed - -- Use correct python version in lightning component template ([#13790](https://github.com/Lightning-AI/lightning/pull/13790)) - -## [0.5.2] - 2022-07-18 - -### Added - -- Update the Lightning App docs ([#13537](https://github.com/Lightning-AI/lightning/pull/13537)) - -### Changed - -- Added `LIGHTNING_` prefix to Platform AWS credentials ([#13703](https://github.com/Lightning-AI/lightning/pull/13703)) diff --git a/src/lightning/app/__init__.py b/src/lightning/app/__init__.py deleted file mode 100644 index 5c904cc4a908c..0000000000000 --- a/src/lightning/app/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Root package info.""" - -import logging -import os - -from lightning_utilities.core.imports import module_available, package_available - -_root_logger = logging.getLogger() -_logger = logging.getLogger(__name__) -_logger.setLevel(logging.INFO) - -_console = logging.StreamHandler() -_console.setLevel(logging.INFO) - -formatter = logging.Formatter("%(levelname)s: %(message)s") -_console.setFormatter(formatter) - -# if root logger has handlers, propagate messages up and let root logger process them, -# otherwise use our own handler -if not _root_logger.hasHandlers(): - _logger.addHandler(_console) - _logger.propagate = False - - -if os.path.isfile(os.path.join(os.path.dirname(__file__), "__about__.py")): - from lightning.app.__about__ import * # noqa: F403 -if "__version__" not in locals(): - if os.path.isfile(os.path.join(os.path.dirname(__file__), "__version__.py")): - from lightning.app.__version__ import version as __version__ - elif package_available("lightning"): - from lightning import __version__ # noqa: F401 - -from lightning.app.core.app import LightningApp # noqa: E402 -from lightning.app.core.flow import LightningFlow # noqa: E402 -from lightning.app.core.work import LightningWork # noqa: E402 -from lightning.app.plugin.plugin import LightningPlugin # noqa: E402 -from lightning.app.utilities.packaging.build_config import BuildConfig # noqa: E402 -from lightning.app.utilities.packaging.cloud_compute import CloudCompute # noqa: E402 - -if module_available("lightning.app.components.demo"): - from lightning.app.components import demo # noqa: F401 - -__package_name__ = "lightning.app".split(".")[0] - -_PACKAGE_ROOT = os.path.dirname(__file__) -_PROJECT_ROOT = os.path.dirname(os.path.dirname(_PACKAGE_ROOT)) -if __package_name__ == "lightning": - _PACKAGE_ROOT = os.path.dirname(_PACKAGE_ROOT) - _PROJECT_ROOT = os.path.dirname(_PROJECT_ROOT) - -__all__ = ["LightningApp", "LightningFlow", "LightningWork", "LightningPlugin", "BuildConfig", "CloudCompute"] diff --git a/src/lightning/app/api/__init__.py b/src/lightning/app/api/__init__.py deleted file mode 100644 index d850e874da5a2..0000000000000 --- a/src/lightning/app/api/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from lightning.app.api.http_methods import Delete, Get, Post, Put - -__all__ = [ - "Delete", - "Get", - "Post", - "Put", -] diff --git a/src/lightning/app/api/http_methods.py b/src/lightning/app/api/http_methods.py deleted file mode 100644 index aa9e68528e487..0000000000000 --- a/src/lightning/app/api/http_methods.py +++ /dev/null @@ -1,258 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import inspect -import time -from copy import deepcopy -from dataclasses import dataclass -from functools import wraps -from multiprocessing import Queue -from typing import Any, Callable, Dict, List, Optional -from uuid import uuid4 - -from fastapi import FastAPI, HTTPException, Request, status -from lightning_utilities.core.apply_func import apply_to_collection - -from lightning.app.api.request_types import _APIRequest, _CommandRequest, _RequestResponse -from lightning.app.utilities.app_helpers import Logger - -logger = Logger(__name__) - - -def _signature_proxy_function(): - pass - - -@dataclass -class _FastApiMockRequest: - """This class is meant to mock FastAPI Request class that isn't pickle-able. - - If a user relies on FastAPI Request annotation, the Lightning framework - patches the annotation before pickling and replace them right after. - - Finally, the FastAPI request is converted back to the _FastApiMockRequest - before being delivered to the users. - - Example: - - from lightning.app import LightningFlow - from fastapi import Request - from lightning.app.api import Post - - class Flow(LightningFlow): - - def request(self, request: Request) -> OutputRequestModel: - ... - - def configure_api(self): - return [Post("/api/v1/request", self.request)] - - """ - - _body: Optional[str] = None - _json: Optional[str] = None - _method: Optional[str] = None - _headers: Optional[Dict] = None - - @property - def receive(self): - raise NotImplementedError - - @property - def method(self): - return self._method - - @property - def headers(self): - return self._headers - - def body(self): - return self._body - - def json(self): - return self._json - - def stream(self): - raise NotImplementedError - - def form(self): - raise NotImplementedError - - def close(self): - raise NotImplementedError - - def is_disconnected(self): - raise NotImplementedError - - -async def _mock_fastapi_request(request: Request): - # TODO: Add more requests parameters. - return _FastApiMockRequest( - _body=await request.body(), - _json=await request.json(), - _headers=request.headers, - _method=request.method, - ) - - -class _HttpMethod: - def __init__( - self, route: str, method: Callable, method_name: Optional[str] = None, timeout: int = 30, **kwargs: Any - ): - """This class is used to inject user defined methods within the App Rest API. - - Arguments: - route: The path used to route the requests - method: The associated flow method - timeout: The time in seconds taken before raising a timeout exception. - - """ - self.route = route - self.attached_to_flow = hasattr(method, "__self__") - self.method_name = method_name or method.__name__ - self.method_annotations = method.__annotations__ - # TODO: Validate the signature contains only pydantic models. - self.method_signature = inspect.signature(method) - - if not self.attached_to_flow: - self.component_name = method.__name__ - self.method = method - else: - self.component_name = method.__self__.name - - self.timeout = timeout - self.kwargs = kwargs - - # Enable the users to rely on FastAPI annotation typing with Request. - # Note: Only a part of the Request functionatilities are supported. - self._patch_fast_api_request() - - def add_route(self, app: FastAPI, request_queue: Queue, responses_store: Dict[str, Any]) -> None: - # 1: Get the route associated with the http method. - route = getattr(app, self.__class__.__name__.lower()) - - self._unpatch_fast_api_request() - - # 2: Create a proxy function with the signature of the wrapped method. - fn = deepcopy(_signature_proxy_function) - fn.__annotations__ = self.method_annotations - fn.__name__ = self.method_name - setattr(fn, "__signature__", self.method_signature) - - # Note: Handle requests differently if attached to a flow. - if not self.attached_to_flow: - # 3: Define the request handler. - @wraps(_signature_proxy_function) - async def _handle_request(*args: Any, **kwargs: Any): - if inspect.iscoroutinefunction(self.method): - return await self.method(*args, **kwargs) - return self.method(*args, **kwargs) - - else: - request_cls = _CommandRequest if self.route.startswith("/command/") else _APIRequest - - # 3: Define the request handler. - @wraps(_signature_proxy_function) - async def _handle_request(*args: Any, **kwargs: Any): - async def fn(*args: Any, **kwargs: Any): - args, kwargs = apply_to_collection((args, kwargs), Request, _mock_fastapi_request) - for k, v in kwargs.items(): - if hasattr(v, "__await__"): - kwargs[k] = await v - - request_id = str(uuid4()).split("-")[0] - logger.debug(f"Processing request {request_id} for route: {self.route}") - request_queue.put( - request_cls( - name=self.component_name, - method_name=self.method_name, - args=args, - kwargs=kwargs, - id=request_id, - ) - ) - - t0 = time.time() - while request_id not in responses_store: - await asyncio.sleep(0.01) - if (time.time() - t0) > self.timeout: - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="The response was never received.", - ) - - logger.debug(f"Processed request {request_id} for route: {self.route}") - - return responses_store.pop(request_id) - - response: _RequestResponse = await asyncio.create_task(fn(*args, **kwargs)) - - if response.status_code != 200: - raise HTTPException(response.status_code, detail=response.content) - - return response.content - - # 4: Register the user provided route to the Rest API. - route(self.route, **self.kwargs)(_handle_request) - - def _patch_fast_api_request(self): - """This function replaces signature annotation for Request with its mock.""" - for k, v in self.method_annotations.items(): - if v == Request: - self.method_annotations[k] = _FastApiMockRequest - - for v in self.method_signature.parameters.values(): - if v._annotation == Request: - v._annotation = _FastApiMockRequest - - def _unpatch_fast_api_request(self): - """This function replaces back signature annotation to fastapi Request.""" - for k, v in self.method_annotations.items(): - if v == _FastApiMockRequest: - self.method_annotations[k] = Request - - for v in self.method_signature.parameters.values(): - if v._annotation == _FastApiMockRequest: - v._annotation = Request - - -class Post(_HttpMethod): - pass - - -class Get(_HttpMethod): - pass - - -class Put(_HttpMethod): - pass - - -class Delete(_HttpMethod): - pass - - -def _add_tags_to_api(apis: List[_HttpMethod], tags: List[str]) -> None: - for api in apis: - if not api.kwargs.get("tag"): - api.kwargs["tags"] = tags - - -def _validate_api(apis: List[_HttpMethod]) -> None: - for api in apis: - if not isinstance(api, _HttpMethod): - raise Exception(f"The provided api should be either [{Delete}, {Get}, {Post}, {Put}]") - if api.route.startswith("/command"): - raise Exception("The route `/command` is reserved for commands. Please, use something else.") diff --git a/src/lightning/app/api/request_types.py b/src/lightning/app/api/request_types.py deleted file mode 100644 index def50e3a20e10..0000000000000 --- a/src/lightning/app/api/request_types.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from dataclasses import asdict, dataclass -from typing import Any, Optional - -from deepdiff import Delta - - -@dataclass -class _BaseRequest: - def to_dict(self): - return asdict(self) - - -@dataclass -class _DeltaRequest(_BaseRequest): - delta: Delta - - def to_dict(self): - return self.delta.to_dict() - - -@dataclass -class _CommandRequest(_BaseRequest): - id: str - name: str - method_name: str - args: Any - kwargs: Any - - -@dataclass -class _APIRequest(_BaseRequest): - id: str - name: str - method_name: str - args: Any - kwargs: Any - - -@dataclass -class _RequestResponse(_BaseRequest): - status_code: int - content: Optional[str] = None diff --git a/src/lightning/app/cli/__init__.py b/src/lightning/app/cli/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/app-template/.gitignore b/src/lightning/app/cli/app-template/.gitignore deleted file mode 100644 index 70ba25888435f..0000000000000 --- a/src/lightning/app/cli/app-template/.gitignore +++ /dev/null @@ -1,157 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -*install-app* - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg - -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Sphinx documentation -docs/_build/ -docs/source/api/ -docs/source/*.md - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.local_env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# PyCharm -.idea/ - -# Lightning logs -lightning_logs -*.gz -.DS_Store -.*_submit.py -.vscode - -MNIST -*.pt -.storage/ -.shared/ -infra -data -coverage.* -# Frontend build artifacts -*lightning/app/ui* -gradio_cached_examples -/docs/source/api_reference/generated/* -examples/my_own_leaderboard/submissions/* -docs/source/api_reference/generated/* -*.ckpt -redis-stable -node_modules -*.rdb -*.webm -*hars -examples/quick_start/* -examples/quick_start -examples/template_react_ui/* -examples/template_react_ui -# Ignore external components -lightning/app/components/* -!lightning/app/components/python -!lightning/app/components/serve -!lightning/app/components/__init__.py -!lightning/app/components/README.md -train_script.py -*return_values* -scratch -storage diff --git a/src/lightning/app/cli/app-template/LICENSE b/src/lightning/app/cli/app-template/LICENSE deleted file mode 100644 index 261eeb9e9f8b2..0000000000000 --- a/src/lightning/app/cli/app-template/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/lightning/app/cli/app-template/README.md b/src/lightning/app/cli/app-template/README.md deleted file mode 100644 index 76c88e6cedb38..0000000000000 --- a/src/lightning/app/cli/app-template/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# placeholdername app - -This ⚡ [Lightning app](https://lightning.ai/) ⚡ was generated automatically with: - -```bash -lightning_app init app placeholdername -``` - -## To run placeholdername - -First, install placeholdername (warning: this app has not been officially approved on the lightning gallery): - -```bash -lightning_app install app https://github.com/theUser/placeholdername -``` - -Once the app is installed, run it locally with: - -```bash -lightning_app run app placeholdername/app.py -``` - -Run it on the [lightning cloud](https://lightning.ai/) with: - -```bash -lightning_app run app placeholdername/app.py --cloud -``` - -## to test and link - -Run flake to make sure all your styling is consistent (it keeps your team from going insane) - -```bash -flake8 . -``` - -To test, follow the README.md instructions in the tests folder. diff --git a/src/lightning/app/cli/app-template/app.py b/src/lightning/app/cli/app-template/app.py deleted file mode 100644 index 4b86551324ccc..0000000000000 --- a/src/lightning/app/cli/app-template/app.py +++ /dev/null @@ -1,16 +0,0 @@ -from lightning.app import LightningApp, LightningFlow -from placeholdername import ComponentA, ComponentB - - -class LitApp(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component_a = ComponentA() - self.component_b = ComponentB() - - def run(self): - self.component_a.run() - self.component_b.run() - - -app = LightningApp(LitApp()) diff --git a/src/lightning/app/cli/app-template/placeholdername/__init__.py b/src/lightning/app/cli/app-template/placeholdername/__init__.py deleted file mode 100644 index cf954823e0315..0000000000000 --- a/src/lightning/app/cli/app-template/placeholdername/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from placeholdername.components.component_a import ComponentA -from placeholdername.components.component_b import ComponentB - -__all__ = ["ComponentA", "ComponentB"] diff --git a/src/lightning/app/cli/app-template/placeholdername/components/component_a/__init__.py b/src/lightning/app/cli/app-template/placeholdername/components/component_a/__init__.py deleted file mode 100644 index 82753954e0e03..0000000000000 --- a/src/lightning/app/cli/app-template/placeholdername/components/component_a/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from placeholdername.components.component_a.component_a import ComponentA - -__all__ = ["ComponentA"] diff --git a/src/lightning/app/cli/app-template/placeholdername/components/component_a/component_a.py b/src/lightning/app/cli/app-template/placeholdername/components/component_a/component_a.py deleted file mode 100644 index e11ff40c299db..0000000000000 --- a/src/lightning/app/cli/app-template/placeholdername/components/component_a/component_a.py +++ /dev/null @@ -1,6 +0,0 @@ -from lightning.app import LightningFlow - - -class ComponentA(LightningFlow): - def run(self): - print("hello from component A") diff --git a/src/lightning/app/cli/app-template/placeholdername/components/component_b/__init__.py b/src/lightning/app/cli/app-template/placeholdername/components/component_b/__init__.py deleted file mode 100644 index 876454576ad90..0000000000000 --- a/src/lightning/app/cli/app-template/placeholdername/components/component_b/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from placeholdername.components.component_b.component_a import ComponentB - -__all__ = ["ComponentB"] diff --git a/src/lightning/app/cli/app-template/placeholdername/components/component_b/component_a.py b/src/lightning/app/cli/app-template/placeholdername/components/component_b/component_a.py deleted file mode 100644 index d80505d986026..0000000000000 --- a/src/lightning/app/cli/app-template/placeholdername/components/component_b/component_a.py +++ /dev/null @@ -1,6 +0,0 @@ -from lightning.app import LightningFlow - - -class ComponentB(LightningFlow): - def run(self): - print("hello from component B") diff --git a/src/lightning/app/cli/app-template/requirements.txt b/src/lightning/app/cli/app-template/requirements.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/app-template/setup.py b/src/lightning/app/cli/app-template/setup.py deleted file mode 100644 index c398ca985f759..0000000000000 --- a/src/lightning/app/cli/app-template/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python - -from setuptools import find_packages, setup - -setup( - name="placeholdername", - version="0.0.0", - description="⚡ Lightning app ⚡ generated with command: lightning init app", - author="", - author_email="", - # REPLACE WITH YOUR OWN GITHUB PROJECT LINK - url="https://github.com/Lightning-AI/lightning-app-template", - install_requires=[], - packages=find_packages(), -) diff --git a/src/lightning/app/cli/app-template/tests/README.md b/src/lightning/app/cli/app-template/tests/README.md deleted file mode 100644 index 85e8c7faa08f9..0000000000000 --- a/src/lightning/app/cli/app-template/tests/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Run tests - -To run the tests: - -```bash -# go to your app folder -cd placeholdername - -# go to tests folder -cd tests - -# install testing deps -pip install -r requirements.txt - -# run tests -pytest . -``` diff --git a/src/lightning/app/cli/app-template/tests/__init__.py b/src/lightning/app/cli/app-template/tests/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/app-template/tests/requirements.txt b/src/lightning/app/cli/app-template/tests/requirements.txt deleted file mode 100644 index 3185d1c44f033..0000000000000 --- a/src/lightning/app/cli/app-template/tests/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -coverage -codecov>=2.1 -pytest>=5.0.0 -pytest-cov -pytest-flake8 -flake8 -check-manifest -twine==4.0.1 diff --git a/src/lightning/app/cli/app-template/tests/test_placeholdername_app.py b/src/lightning/app/cli/app-template/tests/test_placeholdername_app.py deleted file mode 100644 index 6c7743b93ce1e..0000000000000 --- a/src/lightning/app/cli/app-template/tests/test_placeholdername_app.py +++ /dev/null @@ -1,44 +0,0 @@ -r""" -To test a lightning app: -1. Use LightningTestApp which is a subclass of LightningApp. -2. Subclass run_once in LightningTestApp. -3. in run_once, come up with a way to verify the behavior you wanted. - -run_once runs your app through one cycle of the event loop and then terminates -""" - -import io -import os -from contextlib import redirect_stdout - -from lightning.app.testing.testing import LightningTestApp, application_testing - - -class LightningAppTestInt(LightningTestApp): - def run_once(self) -> bool: - f = io.StringIO() - with redirect_stdout(f): - super().run_once() - out = f.getvalue() - assert out == "hello from component A\nhello from component B\n" - return True - - -def test_templatename_app(): - start_dir = os.getcwd() - os.chdir("..") - - cwd = os.getcwd() - cwd = os.path.join(cwd, "placeholdername/app.py") - command_line = [ - cwd, - "--blocking", - "False", - "--open-ui", - "False", - ] - result = application_testing(LightningAppTestInt, command_line) - assert result.exit_code == 0 - - # reset dir - os.chdir(start_dir) diff --git a/src/lightning/app/cli/cmd_apps.py b/src/lightning/app/cli/cmd_apps.py deleted file mode 100644 index d8d7deace2bb4..0000000000000 --- a/src/lightning/app/cli/cmd_apps.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from datetime import datetime -from typing import List, Optional - -from lightning_cloud.openapi import ( - Externalv1LightningappInstance, - Externalv1Lightningwork, - V1LightningappInstanceState, - V1LightningappInstanceStatus, -) -from rich.console import Console -from rich.table import Table -from rich.text import Text - -from lightning.app.cli.core import Formatable -from lightning.app.utilities.cloud import _get_project -from lightning.app.utilities.network import LightningClient - - -class _AppManager: - """_AppManager implements API calls specific to Lightning AI BYOC apps.""" - - def __init__(self) -> None: - self.api_client = LightningClient(retry=False) - - def get_app(self, app_id: str) -> Externalv1LightningappInstance: - project = _get_project(self.api_client) - return self.api_client.lightningapp_instance_service_get_lightningapp_instance( - project_id=project.project_id, id=app_id - ) - - def list_apps(self, limit: int = 100, phase_in: Optional[List[str]] = None) -> List[Externalv1LightningappInstance]: - phase_in = phase_in or [] - project = _get_project(self.api_client) - - kwargs = { - "project_id": project.project_id, - "limit": limit, - "phase_in": phase_in, - } - - resp = self.api_client.lightningapp_instance_service_list_lightningapp_instances(**kwargs) - apps = resp.lightningapps - while resp.next_page_token is not None and resp.next_page_token != "": - kwargs["page_token"] = resp.next_page_token - resp = self.api_client.lightningapp_instance_service_list_lightningapp_instances(**kwargs) - apps = apps + resp.lightningapps - return apps - - def list_components(self, app_id: str, phase_in: Optional[List[str]] = None) -> List[Externalv1Lightningwork]: - phase_in = phase_in or [] - project = _get_project(self.api_client) - resp = self.api_client.lightningwork_service_list_lightningwork( - project_id=project.project_id, - app_id=app_id, - phase_in=phase_in, - ) - return resp.lightningworks - - def list(self, limit: int = 100) -> None: - console = Console() - console.print(_AppList(self.list_apps(limit=limit)).as_table()) - - def delete(self, app_id: str) -> None: - project = _get_project(self.api_client) - self.api_client.lightningapp_instance_service_delete_lightningapp_instance( - project_id=project.project_id, - id=app_id, - ) - - -class _AppList(Formatable): - def __init__(self, apps: List[Externalv1LightningappInstance]) -> None: - self.apps = apps - - @staticmethod - def _textualize_state_transitions( - desired_state: V1LightningappInstanceState, current_state: V1LightningappInstanceStatus - ) -> Text: - phases = { - V1LightningappInstanceState.IMAGE_BUILDING: Text("building image", style="bold yellow"), - V1LightningappInstanceState.PENDING: Text("pending", style="bold yellow"), - V1LightningappInstanceState.RUNNING: Text("running", style="bold green"), - V1LightningappInstanceState.FAILED: Text("failed", style="bold red"), - V1LightningappInstanceState.STOPPED: Text("stopped"), - V1LightningappInstanceState.NOT_STARTED: Text("not started"), - V1LightningappInstanceState.DELETED: Text("deleted", style="bold red"), - V1LightningappInstanceState.UNSPECIFIED: Text("unspecified", style="bold red"), - } - - if current_state.phase == V1LightningappInstanceState.UNSPECIFIED and current_state.start_timestamp is None: - return Text("not yet started", style="bold yellow") - - if ( - desired_state == V1LightningappInstanceState.DELETED - and current_state.phase != V1LightningappInstanceState.DELETED - ): - return Text("terminating", style="bold red") - - if ( - any( - phase == current_state.phase - for phase in [V1LightningappInstanceState.PENDING, V1LightningappInstanceState.STOPPED] - ) - and desired_state == V1LightningappInstanceState.RUNNING - ): - return Text("restarting", style="bold yellow") - - return phases[current_state.phase] - - def as_json(self) -> str: - return json.dumps(self.apps) - - def as_table(self) -> Table: - table = Table("id", "name", "status", "created", show_header=True, header_style="bold green") - - for app in self.apps: - status = self._textualize_state_transitions(desired_state=app.spec.desired_state, current_state=app.status) - - # this guard is necessary only until 0.3.93 releases which includes the `created_at` - # field to the external API - created_at = datetime.now() - if hasattr(app, "created_at"): - created_at = app.created_at - - table.add_row( - app.id, - app.name, - status, - created_at.strftime("%Y-%m-%d") if created_at else "", - ) - return table diff --git a/src/lightning/app/cli/cmd_init.py b/src/lightning/app/cli/cmd_init.py deleted file mode 100644 index db83fd41e47d9..0000000000000 --- a/src/lightning/app/cli/cmd_init.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shutil -from typing import List, Optional, Tuple - -from lightning.app.utilities.app_helpers import Logger - -logger = Logger(__name__) - - -def app(app_name: str) -> None: - if app_name is None: - app_name = _capture_valid_app_component_name(resource_type="app") - - # generate resource template - new_resource_name, _ = _make_resource(resource_dir="app-template", resource_name=app_name) - - m = f""" - ⚡ Lightning app template created! ⚡ - {new_resource_name} - - run your app with: - lightning run app {app_name}/app.py - - run it on the cloud to share with your collaborators: - lightning run app {app_name}/app.py --cloud - """ - logger.info(m) - - -def _make_resource(resource_dir: str, resource_name: str) -> Tuple[str, str]: - path = os.path.dirname(os.path.abspath(__file__)) - template_dir = os.path.join(path, resource_dir) - name_for_files = re.sub("-", "_", resource_name) - - new_resource_name = os.path.join(os.getcwd(), resource_name) - - # lay out scaffolding - logger.info(f"laying out component template at {new_resource_name}") - shutil.copytree(template_dir, new_resource_name) - - # rename main folder - os.rename(os.path.join(new_resource_name, "placeholdername"), os.path.join(new_resource_name, name_for_files)) - - # for each file, rename the word - trouble_names = {".DS_Store"} - files = _ls_recursively(new_resource_name) - for bad_file in files: - if bad_file.split("/")[-1] in trouble_names: - continue - # find the words and replace - with open(bad_file) as fo: - content = fo.read().replace("placeholdername", name_for_files) - with open(bad_file, "w") as fw: - fw.write(content) - - # rename files - for file_name in files: - new_file = re.sub("placeholdername", name_for_files, file_name) - os.rename(file_name, new_file) - - return new_resource_name, name_for_files - - -def _ls_recursively(dir_name: str) -> List[str]: - fname = [] - for root, d_names, f_names in os.walk(dir_name): - for f in f_names: - if "__pycache__" not in root: - fname.append(os.path.join(root, f)) - - return fname - - -def _capture_valid_app_component_name(value: Optional[str] = None, resource_type: str = "app") -> str: - prompt = f""" - ⚡ Creating Lightning {resource_type} ⚡ - """ - logger.info(prompt) - - try: - if value is None: - value = input(f"\nName your Lightning {resource_type} (example: the-{resource_type}-name) > ") - value = value.strip().lower() - unsafe_chars = set(re.findall(r"[^a-z0-9\-]", value)) - if len(unsafe_chars) > 0: - m = f""" - Error: your Lightning {resource_type} name: - {value} - - contains the following unsupported characters: - {unsafe_chars} - - A Lightning {resource_type} name can only contain letters (a-z) numbers (0-9) and the '-' character - - valid example: - lightning-{resource_type} - """ - raise SystemExit(m) - - except KeyboardInterrupt: - raise SystemExit( - f""" - ⚡ {resource_type} init aborted! ⚡ - """ - ) - - return value - - -def component(component_name: str) -> None: - if component_name is None: - component_name = _capture_valid_app_component_name(resource_type="component") - - # generate resource template - new_resource_name, name_for_files = _make_resource(resource_dir="component-template", resource_name=component_name) - - m = f""" - ⚡ Lightning component template created! ⚡ - {new_resource_name} - - ⚡ To use your component, first pip install it (with these 3 commands): ⚡ - cd {component_name} - pip install -r requirements.txt - pip install -e . - - ⚡ Use the component inside an app: ⚡ - - from {name_for_files} import TemplateComponent - import lightning.app as la - - class LitApp(la.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.{name_for_files} = TemplateComponent() - - def run(self): - print('this is a simple Lightning app to verify your component is working as expected') - self.{name_for_files}.run() - - app = la.LightningApp(LitApp()) - - ⚡ Checkout the demo app with your {component_name} component: ⚡ - lightning run app {component_name}/app.py - - ⚡ Tip: Publish your component to the Lightning Gallery to enable users to install it like so: - lightning install component YourLightningUserName/{component_name} - - so the Lightning community can use it like: - from {name_for_files} import TemplateComponent - - """ - logger.info(m) diff --git a/src/lightning/app/cli/cmd_install.py b/src/lightning/app/cli/cmd_install.py deleted file mode 100644 index b43aa3f88fac9..0000000000000 --- a/src/lightning/app/cli/cmd_install.py +++ /dev/null @@ -1,657 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shutil -import subprocess -import sys -from typing import Dict, Optional, Tuple - -import click -import requests -from packaging.version import Version - -from lightning.app.core.constants import LIGHTNING_APPS_PUBLIC_REGISTRY, LIGHTNING_COMPONENT_PUBLIC_REGISTRY -from lightning.app.utilities.app_helpers import Logger - -logger = Logger(__name__) - - -@click.group(name="install") -def install() -> None: - """Install Lightning AI selfresources.""" - pass - - -@install.command("app") -@click.argument("name", type=str) -@click.option( - "--yes", - "-y", - is_flag=True, - help="disables prompt to ask permission to create env and run install cmds", -) -@click.option( - "--version", - "-v", - type=str, - help="Specify the version to install. By default it uses 'latest'", - default="latest", - show_default=True, -) -@click.option( - "--overwrite", - "-f", - is_flag=True, - default=False, - help="When set, overwrite the app directory without asking if it already exists.", -) -def install_app(name: str, yes: bool, version: str, overwrite: bool = False) -> None: - _install_app_command(name, yes, version, overwrite=overwrite) - - -@install.command("component") -@click.argument("name", type=str) -@click.option( - "--yes", - "-y", - is_flag=True, - help="disables prompt to ask permission to create env and run install cmds", -) -@click.option( - "--version", - "-v", - type=str, - help="Specify the version to install. By default it uses 'latest'", - default="latest", - show_default=True, -) -def install_component(name: str, yes: bool, version: str) -> None: - _install_component_command(name, yes, version) - - -def _install_app_command(name: str, yes: bool, version: str, overwrite: bool = False) -> None: - if "github.com" in name: - if version != "latest": - logger.warn( - "When installing from GitHub, only the 'latest' version is supported. " - f"The provided version ({version}) will be ignored." - ) - return non_gallery_app(name, yes, overwrite=overwrite) - - return gallery_app(name, yes, version, overwrite=overwrite) - - -def _install_component_command(name: str, yes: bool, version: str, overwrite: bool = False) -> None: - if "github.com" in name: - if version != "latest": - logger.warn( - "When installing from GitHub, only the 'latest' version is supported. " - f"The provided version ({version}) will be ignored." - ) - return non_gallery_component(name, yes) - - return gallery_component(name, yes, version) - - -def gallery_apps_and_components( - name: str, yes_arg: bool, version_arg: str, cwd: Optional[str] = None, overwrite: bool = False -) -> Optional[str]: - try: - org, app_or_component = name.split("/") - except Exception: - return None - - entry, kind = _resolve_entry(name, version_arg) - - if kind == "app": - # give the user the chance to do a manual install - source_url, git_url, folder_name, git_sha = _show_install_app_prompt( - entry, app_or_component, org, yes_arg, resource_type="app" - ) - # run installation if requested - _install_app_from_source(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha) - - return os.path.join(os.getcwd(), *entry["appEntrypointFile"].split("/")) - - if kind == "component": - # give the user the chance to do a manual install - source_url, git_url, folder_name, git_sha = _show_install_app_prompt( - entry, app_or_component, org, yes_arg, resource_type="component" - ) - if "@" in git_url: - git_url = git_url.split("git+")[1].split("@")[0] - # run installation if requested - _install_app_from_source(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha) - - return os.path.join(os.getcwd(), *entry["entrypointFile"].split("/")) - - return None - - -def gallery_component(name: str, yes_arg: bool, version_arg: str, cwd: Optional[str] = None) -> str: - # make sure org/component-name name is correct - org, component = _validate_name(name, resource_type="component", example="lightning/LAI-slack-component") - - # resolve registry (orgs can have a private registry through their environment variables) - registry_url = _resolve_component_registry() - - # load the component resource - component_entry = _resolve_resource(registry_url, name=name, version_arg=version_arg, resource_type="component") - - # give the user the chance to do a manual install - git_url = _show_install_component_prompt(component_entry, component, org, yes_arg) - - # run installation if requested - _install_component_from_source(git_url) - - return os.path.join(os.getcwd(), component_entry["entrypointFile"]) - - -def non_gallery_component(gh_url: str, yes_arg: bool, cwd: Optional[str] = None) -> None: - # give the user the chance to do a manual install - git_url = _show_non_gallery_install_component_prompt(gh_url, yes_arg) - - # run installation if requested - _install_component_from_source(git_url) - - -def gallery_app(name: str, yes_arg: bool, version_arg: str, cwd: Optional[str] = None, overwrite: bool = False) -> str: - # make sure org/app-name syntax is correct - org, app = _validate_name(name, resource_type="app", example="lightning/quick-start") - - # resolve registry (orgs can have a private registry through their environment variables) - registry_url = _resolve_app_registry() - - # load the app resource - app_entry = _resolve_resource(registry_url, name=name, version_arg=version_arg, resource_type="app") - - # give the user the chance to do a manual install - source_url, git_url, folder_name, git_sha = _show_install_app_prompt( - app_entry, app, org, yes_arg, resource_type="app" - ) - - # run installation if requested - _install_app_from_source(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha) - - return os.path.join(os.getcwd(), folder_name, app_entry["appEntrypointFile"]) - - -def non_gallery_app(gh_url: str, yes_arg: bool, cwd: Optional[str] = None, overwrite: bool = False) -> None: - # give the user the chance to do a manual install - repo_url, folder_name = _show_non_gallery_install_app_prompt(gh_url, yes_arg) - - # run installation if requested - _install_app_from_source(repo_url, repo_url, folder_name, cwd=cwd, overwrite=overwrite) - - -def _show_install_component_prompt(entry: Dict[str, str], component: str, org: str, yes_arg: bool) -> str: - git_url = entry["gitUrl"] - - # yes arg does not prompt the user for permission to install anything - # automatically creates env and sets up the project - if yes_arg: - return git_url - - prompt = f""" - ⚡ Installing Lightning component ⚡ - - component name : {component} - developer : {org} - - Installation runs the following command for you: - - pip install {git_url} - """ - logger.info(prompt) - - try: - value = input("\nPress enter to continue: ") - value = value.strip().lower() - should_install = len(value) == 0 or value in {"y", "yes", 1} - if not should_install: - raise KeyboardInterrupt() - - return git_url - except KeyboardInterrupt: - repo = entry["sourceUrl"] - raise SystemExit( - f""" - ⚡ Installation aborted! ⚡ - - Install the component yourself by visiting: - {repo} - """ - ) - - -def _show_non_gallery_install_component_prompt(gh_url: str, yes_arg: bool) -> str: - if ".git@" not in gh_url: - m = """ - Error, your github url must be in the following format: - git+https://github.com/OrgName/repo-name.git@ALongCommitSHAString - - Example: - git+https://github.com/Lightning-AI/LAI-slack-messenger.git@14f333456ffb6758bd19458e6fa0bf12cf5575e1 - """ - raise SystemExit(m) - - developer = gh_url.split("/")[3] - component_name = gh_url.split("/")[4].split(".git")[0] - repo_url = re.search(r"git\+(.*).git", gh_url).group(1) # type: ignore - - # yes arg does not prompt the user for permission to install anything - # automatically creates env and sets up the project - if yes_arg: - return gh_url - - prompt = f""" - ⚡ Installing Lightning component ⚡ - - component name : {component_name} - developer : {developer} - - ⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡ - WARNING: this is NOT an official Lightning Gallery component - Install at your own risk - ⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡ - - Installation runs the following command for you: - - pip install {gh_url} - """ - logger.info(prompt) - - try: - value = input("\nPress enter to continue: ") - value = value.strip().lower() - should_install = len(value) == 0 or value in {"y", "yes", 1} - if not should_install: - raise KeyboardInterrupt() - - return gh_url - except KeyboardInterrupt: - raise SystemExit( - f""" - ⚡ Installation aborted! ⚡ - - Install the component yourself by visiting: - {repo_url} - """ - ) - - -def _show_install_app_prompt( - entry: Dict[str, str], app: str, org: str, yes_arg: bool, resource_type: str -) -> Tuple[str, str, str, Optional[str]]: - source_url = entry["sourceUrl"] # This URL is used only to display the repo and extract folder name - full_git_url = entry["gitUrl"] # Used to clone the repo (can include tokens for private repos) - git_url_parts = full_git_url.split("#ref=") - git_url = git_url_parts[0] - git_sha = git_url_parts[1] if len(git_url_parts) == 2 else None - - folder_name = source_url.split("/")[-1] - - # yes arg does not prompt the user for permission to install anything - # automatically creates env and sets up the project - if yes_arg: - return source_url, git_url, folder_name, git_sha - - prompt = f""" - ⚡ Installing Lightning {resource_type} ⚡ - - {resource_type} name : {app} - developer: {org} - - Installation creates and runs the following commands for you: - - git clone {source_url} - cd {folder_name} - pip install -r requirements.txt - pip install -e . - """ - logger.info(prompt) - - try: - value = input("\nPress enter to continue: ") - value = value.strip().lower() - should_install = len(value) == 0 or value in {"y", "yes", 1} - if not should_install: - raise KeyboardInterrupt() - - return source_url, git_url, folder_name, git_sha - except KeyboardInterrupt: - repo = entry["sourceUrl"] - raise SystemExit( - f""" - ⚡ Installation aborted! ⚡ - - Install the {resource_type} yourself by visiting: - {repo} - """ - ) - - -def _show_non_gallery_install_app_prompt(gh_url: str, yes_arg: bool) -> Tuple[str, str]: - try: - if gh_url.endswith(".git"): - # folder_name when it's a GH url with .git - folder_name = gh_url.split("/")[-1] - folder_name = folder_name[:-4] - else: - # the last part of the url is the folder name otherwise - folder_name = gh_url.split("/")[-1] - - org = re.search(r"github.com\/(.*)\/", gh_url).group(1) # type: ignore - except Exception: - raise SystemExit( - """ - Your github url is not supported. Here's the supported format: - https://github.com/YourOrgName/your-repo-name - - Example: - https://github.com/Lightning-AI/lightning - """ - ) - - # yes arg does not prompt the user for permission to install anything - # automatically creates env and sets up the project - if yes_arg: - return gh_url, folder_name - - prompt = f""" - ⚡ Installing Lightning app ⚡ - - app source : {gh_url} - developer : {org} - - ⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡ - WARNING: this is NOT an official Lightning Gallery app - Install at your own risk - ⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡ - - Installation creates and runs the following commands for you: - - git clone {gh_url} - cd {folder_name} - pip install -r requirements.txt - pip install -e . - """ - logger.info(prompt) - - try: - value = input("\nPress enter to continue: ") - value = value.strip().lower() - should_install = len(value) == 0 or value in {"y", "yes", 1} - if not should_install: - raise KeyboardInterrupt() - - return gh_url, folder_name - except KeyboardInterrupt: - raise SystemExit( - f""" - ⚡ Installation aborted! ⚡ - - Install the app yourself by visiting {gh_url} - """ - ) - - -def _validate_name(name: str, resource_type: str, example: str) -> Tuple[str, str]: - # ensure resource identifier is properly formatted - try: - org, resource = name.split("/") - except Exception: - raise SystemExit( - f""" - {resource_type} name format must have organization/{resource_type}-name - - Examples: - {example} - user/{resource_type}-name - - You passed in: {name} - """ - ) - return org, resource - - -def _resolve_entry(name, version_arg) -> Tuple[Optional[Dict], Optional[str]]: - entry = None - kind = None - - # resolve registry (orgs can have a private registry through their environment variables) - registry_url = _resolve_app_registry() - - # load the app resource - entry = _resolve_resource(registry_url, name=name, version_arg=version_arg, resource_type="app", raise_error=False) - - if not entry: - registry_url = _resolve_component_registry() - - # load the component resource - entry = _resolve_resource( - registry_url, name=name, version_arg=version_arg, resource_type="component", raise_error=False - ) - kind = "component" if entry else None - - else: - kind = "app" - - return entry, kind - - -def _resolve_resource( - registry_url: str, name: str, version_arg: str, resource_type: str, raise_error: bool = True -) -> Dict[str, str]: - gallery_entries = [] - try: - response = requests.get(registry_url) - data = response.json() - - if resource_type == "app": - gallery_entries = [a for a in data["apps"] if a["canDownloadSourceCode"]] - - elif resource_type == "component": - gallery_entries = data["components"] - except requests.ConnectionError: - sys.tracebacklimit = 0 - raise SystemError( - f""" - Network connection error, could not load list of available Lightning {resource_type}s. - - Try again when you have a network connection! - """ - ) - - entries = [] - all_versions = [] - for x in gallery_entries: - if name == x["name"]: - entries.append(x) - all_versions.append(x["version"]) - - if len(entries) == 0: - if raise_error: - raise SystemExit(f"{resource_type}: '{name}' is not available on ⚡ Lightning AI ⚡") - - return None - - entry = None - if version_arg == "latest": - entry = max(entries, key=lambda app: Version(app["version"])) - else: - for e in entries: - if e["version"] == version_arg: - entry = e - break - if entry is None and raise_error: - if raise_error: - raise Exception( - f"{resource_type}: 'Version {version_arg} for {name}' is not available on ⚡ Lightning AI ⚡. " - f"Here is the list of all availables versions:{os.linesep}{os.linesep.join(all_versions)}" - ) - return None - - return entry - - -def _install_with_env(repo_url: str, folder_name: str, cwd: Optional[str] = None) -> None: - if not cwd: - cwd = os.getcwd() - - # clone repo - logger.info(f"⚡ RUN: git clone {repo_url}") - subprocess.call(["git", "clone", repo_url]) - - # step into the repo folder - os.chdir(f"{folder_name}") - cwd = os.getcwd() - - # create env - logger.info(f"⚡ CREATE: virtual env at {cwd}") - subprocess.call(["python", "-m", "venv", cwd]) - - # activate and install reqs - # TODO: remove shell=True... but need to run command in venv - logger.info("⚡ RUN: install requirements (pip install -r requirements.txt)") - subprocess.call("source bin/activate && pip install -r requirements.txt", shell=True) - - # install project - # TODO: remove shell=True... but need to run command in venv - logger.info("⚡ RUN: setting up project (pip install -e .)") - subprocess.call("source bin/activate && pip install -e .", shell=True) - - m = f""" - ⚡ Installed! ⚡ to use your app - go into the folder: cd {folder_name} - activate the environment: source bin/activate - run the app: lightning run app [the_app_file.py] - """ - logger.info(m) - - -def _install_app_from_source( - source_url: str, - git_url: str, - folder_name: str, - cwd: Optional[str] = None, - overwrite: bool = False, - git_sha: Optional[str] = None, -) -> None: - """Installing lighting app from the `git_url` - - Args: - source_url: - source repo url without any tokens and params, this param is used only for displaying - git_url: - repo url that is used to clone, this can contain tokens - folder_name: - where to clone the repo ? - cwd: - Working director. If not specified, current working directory is used. - overwrite: - If true, overwrite the app directory without asking if it already exists - git_sha: - The git_sha for checking out the git repo of the app. - - """ - - if not cwd: - cwd = os.getcwd() - - destination = os.path.join(cwd, folder_name) - if os.path.exists(destination): - if not overwrite: - raise SystemExit( - f"Folder {folder_name} exists, please delete it and try again, " - f"or force to overwrite the existing folder by passing `--overwrite`.", - ) - shutil.rmtree(destination) - # clone repo - logger.info(f"⚡ RUN: git clone {source_url}") - try: - subprocess.check_output(["git", "clone", git_url], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as ex: - if "Repository not found" in str(ex.output): - raise SystemExit( - f""" - Looks like the github url was not found or doesn't exist. Do you have a typo? - {source_url} - """ - ) - raise Exception(ex) - - # step into the repo folder - os.chdir(f"{folder_name}") - cwd = os.getcwd() - - try: - if git_sha: - subprocess.check_output(["git", "checkout", git_sha], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as ex: - if "did not match any" in str(ex.output): - raise SystemExit("Looks like the git SHA is not valid or doesn't exist in app repo.") - raise Exception(ex) - - # activate and install reqs - # TODO: remove shell=True... but need to run command in venv - logger.info("⚡ RUN: install requirements (pip install -r requirements.txt)") - subprocess.call("pip install -r requirements.txt", shell=True) - - # install project - # TODO: remove shell=True... but need to run command in venv - logger.info("⚡ RUN: setting up project (pip install -e .)") - subprocess.call("pip install -e .", shell=True) - - m = f""" - ⚡ Installed! ⚡ to use your app: - - cd {folder_name} - lightning run app app.py - """ - logger.info(m) - - -def _install_component_from_source(git_url: str) -> None: - logger.info("⚡ RUN: pip install") - - out = subprocess.check_output(["pip", "install", git_url]) - possible_success_message = [x for x in str(out).split("\\n") if "Successfully installed" in x] - if len(possible_success_message) > 0: - uninstall_step = possible_success_message[0] - uninstall_step = re.sub("Successfully installed", "", uninstall_step).strip() - uninstall_step = re.sub("-0.0.0", "", uninstall_step).strip() - m = """ - ⚡ Installed! ⚡ - - to use your component: - from the_component import TheClass - - make sure to add this entry to your Lightning APP requirements.txt file: - {git_url} - - if you want to uninstall, run this command: - pip uninstall {uninstall_step} - """ - logger.info(m) - - -def _resolve_app_registry() -> str: - return os.environ.get("LIGHTNING_APP_REGISTRY", LIGHTNING_APPS_PUBLIC_REGISTRY) - - -def _resolve_component_registry() -> str: - return os.environ.get("LIGHTNING_COMPONENT_REGISTRY", LIGHTNING_COMPONENT_PUBLIC_REGISTRY) diff --git a/src/lightning/app/cli/cmd_pl_init.py b/src/lightning/app/cli/cmd_pl_init.py deleted file mode 100644 index 2436c28179ef2..0000000000000 --- a/src/lightning/app/cli/cmd_pl_init.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib -import re -import shutil -import subprocess -import sys -import tarfile -import urllib.request -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Any, Dict, List, Optional - -import click -from jinja2 import Environment, FileSystemLoader -from rich import print -from rich.panel import Panel -from rich.status import Status -from rich.text import Text -from rich.tree import Tree - -import lightning.app - -_REPORT_HELP_TEXTS = { - "core": "Important files for the app such as various components", - "source": "A copy of all your source code, including the PL script ⚡", - "tests": "This app comes with tests!", - "ui": "Source and build files for the user interface", - "app.py": "This is the main app file!", - "requirements.txt": "Lists the dependencies required to be installed before running the app", -} - -_REPORT_IGNORE_PATTERNS = [ - r"__pycache__", - r"__init__\.py", - r".*egg-info", - r"\..*", -] - - -def pl_app(source_dir: str, script_path: str, name: str, overwrite: bool) -> None: - source_dir = Path(source_dir).resolve() - script_path = Path(script_path).resolve() - - if not source_dir.is_dir(): - click.echo(f"The given source directory does not exist: {source_dir}", err=True) - raise SystemExit(1) - - if not script_path.exists(): - click.echo(f"The given script path does not exist: {script_path}", err=True) - raise SystemExit(1) - - if not script_path.is_file(): - click.echo(f"The given script path must be a file, you passed: {script_path}", err=True) - raise SystemExit(1) - - if source_dir not in script_path.parents: - click.echo( - "The given script path must be a subpath of the source directory. Example:" - " lightning init pl-app ./code ./code/scripts/train.py", - err=True, - ) - raise SystemExit(1) - - rel_script_path = script_path.relative_to(source_dir) - cwd = Path.cwd() - destination = cwd / name - - if destination.exists(): - if not overwrite: - click.echo( - f"There is already an app with the name {name} in the current working directory. Choose a different" - f" name with `--name` or force to overwrite the existing folder by passing `--overwrite`.", - err=True, - ) - raise SystemExit(1) - - shutil.rmtree(destination) - - template_dir = Path(lightning.app.cli.__file__).parent / "pl-app-template" - - with Status("[bold green]Copying app files"): - shutil.copytree(template_dir, destination, ignore=shutil.ignore_patterns("node_modules", "build")) - if (template_dir / "ui" / "build").exists(): - shutil.copytree(template_dir / "ui" / "build", destination / "ui" / "build") - else: - download_frontend(destination / "ui" / "build") - - with Status("[bold green]Copying source files"): - shutil.copytree(source_dir, destination / "source", ignore=shutil.ignore_patterns(name)) - project_file_from_template(template_dir, destination, "app.py", script_path=str(rel_script_path)) - project_file_from_template(template_dir, destination, "setup.py", app_name=name) - - with Status("[bold green]Installing"): - subprocess.call(["pip", "install", "--quiet", "-e", str(destination)]) - # TODO: download the ui files - - print_pretty_report( - destination, - ignore_patterns=_REPORT_IGNORE_PATTERNS, - help_texts=_REPORT_HELP_TEXTS, - ) - - -def download_frontend(destination: Path) -> None: - # TODO: Update the URL to the release in GitHub once the PL app repo is public - url = "https://storage.googleapis.com/grid-packages/pytorch-lightning-app/v0.0.0/build.tar.gz" - build_dir_name = "build" - with TemporaryDirectory() as download_dir: - response = urllib.request.urlopen(url) # noqa: S310 - file = tarfile.open(fileobj=response, mode="r|gz") - file.extractall(path=download_dir) # noqa: S202 - shutil.move(str(Path(download_dir, build_dir_name)), destination) - - -def project_file_from_template(template_dir: Path, destination_dir: Path, template_name: str, **kwargs: Any) -> None: - env = Environment(loader=FileSystemLoader(template_dir)) # noqa: S701 - template = env.get_template(template_name) - rendered_template = template.render(**kwargs) - with open(destination_dir / template_name, "w") as file: - file.write(rendered_template) - - -def print_pretty_report( - directory: pathlib.Path, - ignore_patterns: Optional[List[str]] = None, - help_texts: Optional[Dict[str, str]] = None, -) -> None: - """Prints a report for the generated app.""" - tree = Tree( - f":open_file_folder: [link file://{directory}]{directory}", - guide_style="bold bright_blue", - ) - - help_texts = {} if help_texts is None else help_texts - - paths = sorted( - directory.glob("*"), - key=lambda p: (p.is_file(), p.name.lower()), - ) - max_witdth = max(len(p.name) for p in paths) - - patterns_to_ignore = [] if ignore_patterns is None else ignore_patterns - for path in paths: - if any(re.match(pattern, path.name) for pattern in patterns_to_ignore): - # Only display relevant files - continue - - help_text = help_texts.get(path.name, "") - padding = " " * (max_witdth - len(path.name)) - - text_pathname = Text(path.name, "green") - text_pathname.highlight_regex(r"\..*$", "bold red") - text_pathname.stylize(f"link file://{path}") - text_pathname.append(f" {padding} {help_text}", "blue") - - icon = "📂 " if path.is_dir() else "📄 " - icon = icon if _can_encode_icon(icon) else "" - - tree.add(Text(icon) + text_pathname) - - print("\n") - print("Done. The app is ready here:\n") - print(tree) - print("\nRun it:\n") - print(Panel(f"[red]lightning run app {directory.relative_to(Path.cwd()) / 'app.py'}")) - - -def _can_encode_icon(icon: str) -> bool: - """Helper function to check whether an icon can be encoded.""" - try: - icon.encode(sys.stdout.encoding) - return True - except UnicodeEncodeError: - return False diff --git a/src/lightning/app/cli/cmd_react_ui_init.py b/src/lightning/app/cli/cmd_react_ui_init.py deleted file mode 100644 index 22e668433e233..0000000000000 --- a/src/lightning/app/cli/cmd_react_ui_init.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shutil -import subprocess -from typing import Optional - -from lightning.app.utilities.app_helpers import Logger - -logger = Logger(__name__) - - -def react_ui(dest_dir: Optional[str] = None) -> None: - # verify all the prereqs for install are met - _check_react_prerequisites() - - # copy template files to the dir - _copy_and_setup_react_ui(dest_dir) - - -def _copy_and_setup_react_ui(dest_dir: Optional[str] = None) -> None: - logger.info("⚡ setting up react-ui template") - path = os.path.dirname(os.path.abspath(__file__)) - template_dir = os.path.join(path, "react-ui-template") - - if dest_dir is None: - dest_dir = os.path.join(os.getcwd(), "react-ui") - - shutil.copytree(template_dir, dest_dir) - - logger.info("⚡ install react project deps") - ui_path = os.path.join(dest_dir, "ui") - subprocess.run(f"cd {ui_path} && yarn install", shell=True) - - logger.info("⚡ building react project") - subprocess.run(f"cd {ui_path} && yarn build", shell=True) - - m = f""" - ⚡⚡ react-ui created! ⚡⚡ - - ⚡ Connect it to your component using `configure_layout`: - - # Use a LightningFlow or LightningWork - class YourComponent(la.LightningFlow): - def configure_layout(self): - return la.frontend.StaticWebFrontend(Path(__file__).parent / "react-ui/src/dist") - - ⚡ run the example_app.py to see it live! - lightning_app run app {dest_dir}/example_app.py - - """ - logger.info(m) - - -def _check_react_prerequisites() -> None: - """Args are for test purposes only.""" - missing_msgs = [] - version_regex = r"\d{1,2}\.\d{1,2}\.\d{1,3}" - - logger.info("Checking pre-requisites for react") - - # make sure npm is installed - npm_version = subprocess.check_output(["npm", "--version"]) - has_npm = bool(re.search(version_regex, str(npm_version))) - npm_version = re.search(version_regex, str(npm_version)) - npm_version = None if npm_version is None else npm_version.group(0) - - if not has_npm: - m = """ - This machine is missing 'npm'. Please install npm and rerun 'lightning_app init react-ui' again. - - Install instructions: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm - """ - missing_msgs.append(m) - - # make sure node is installed - node_version = subprocess.check_output(["node", "--version"]) - has_node = bool(re.search(version_regex, str(node_version))) - node_version = re.search(version_regex, str(node_version)) - node_version = None if node_version is None else node_version.group(0) - - if not has_node: - m = """ - This machine is missing 'node'. Please install node and rerun 'lightning_app init react-ui' again. - - Install instructions: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm - """ - missing_msgs.append(m) - - # make sure yarn is installed - yarn_version = subprocess.check_output(["yarn", "--version"]) - has_yarn = bool(re.search(version_regex, str(yarn_version))) - yarn_version = re.search(version_regex, str(yarn_version)) - yarn_version = None if yarn_version is None else yarn_version.group(0) - - if not has_yarn: - m = """ - This machine is missing 'yarn'. Please install npm+node first, then run - - npm install --global yarn - - Full install instructions: https://classic.yarnpkg.com/lang/en/docs/install/#mac-stable - """ - missing_msgs.append(m) - - # exit or show success message - if len(missing_msgs) > 0: - missing_msg = "\n".join(missing_msgs) - raise SystemExit(missing_msg) - logger.info( - f""" - found npm version: {npm_version} - found node version: {node_version} - found yarn version: {yarn_version} - - Pre-requisites met! - """ - ) diff --git a/src/lightning/app/cli/commands/__init__.py b/src/lightning/app/cli/commands/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/commands/app_commands.py b/src/lightning/app/cli/commands/app_commands.py deleted file mode 100644 index bbecceabb6e28..0000000000000 --- a/src/lightning/app/cli/commands/app_commands.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -from typing import Dict, Optional - -import requests - -from lightning.app.cli.connect.app import ( - _clean_lightning_connection, - _install_missing_requirements, - _resolve_command_path, -) -from lightning.app.utilities.cli_helpers import _LightningAppOpenAPIRetriever -from lightning.app.utilities.commands.base import _download_command -from lightning.app.utilities.enum import OpenAPITags - - -def _is_running_help(argv) -> bool: - return argv[-1] in ["--help", "-"] if argv else False - - -def _run_app_command(app_name: str, app_id: Optional[str]): - """Execute a function in a running App from its name.""" - # 1: Collect the url and comments from the running application - _clean_lightning_connection() - - running_help = _is_running_help(sys.argv) - - retriever = _LightningAppOpenAPIRetriever(app_id, use_cache=running_help) - - if not running_help and (retriever.url is None or retriever.api_commands is None): - if app_name == "localhost": - print("The command couldn't be executed as your local Lightning App isn't running.") - else: - print(f"The command couldn't be executed as your cloud Lightning App `{app_name}` isn't running.") - sys.exit(0) - - if not retriever.api_commands: - raise Exception("This application doesn't expose any commands yet.") - - full_command = "_".join(sys.argv) - - has_found = False - for command in list(retriever.api_commands): - if command in full_command: - has_found = True - for value in sys.argv: - if value == command and "_" in value: - print( - f"The command `{value}` was provided with an underscore and it isn't allowed." - f"Instead, use `lightning_app {value.replace('_', ' ')}`." - ) - sys.exit(0) - break - - if not has_found: - raise Exception(f"The provided command isn't available in {list(retriever.api_commands)}") - - # 2: Send the command from the user - metadata = retriever.api_commands[command] - - try: - # 3: Execute the command - if metadata["tag"] == OpenAPITags.APP_COMMAND: - _handle_command_without_client(command, metadata, retriever.url) - else: - _handle_command_with_client(command, metadata, app_name, app_id, retriever.url) - except ModuleNotFoundError: - _install_missing_requirements(retriever, fail_if_missing=True) - - if running_help: - print("Your command execution was successful.") - - -def _handle_command_without_client(command: str, metadata: Dict, url: str) -> None: - supported_params = list(metadata["parameters"]) - if _is_running_help(sys.argv): - print(f"Usage: lightning_app {command} [ARGS]...") - print(" ") - print("Options") - for param in supported_params: - print(f" {param}: Add description") - return - - provided_params = [param.replace("--", "") for param in sys.argv[1 + len(command.split("_")) :]] - - # TODO: Add support for more argument types. - if any("=" not in param for param in provided_params): - raise Exception("Please, use --x=y syntax when providing the command arguments.") - - if any(param.split("=")[0] not in supported_params for param in provided_params): - raise Exception(f"Some arguments need to be provided. The keys are {supported_params}.") - - # TODO: Encode the parameters and validate their type. - query_parameters = "&".join(provided_params) - resp = requests.post(url + f"/command/{command}?{query_parameters}") - assert resp.status_code == 200, resp.json() - print(resp.json()) - - -def _handle_command_with_client(command: str, metadata: Dict, app_name: str, app_id: Optional[str], url: str): - debug_mode = bool(int(os.getenv("DEBUG", "0"))) - - if app_name == "localhost": - target_file = metadata["cls_path"] - else: - target_file = _resolve_command_path(command) if debug_mode else _resolve_command_path(command) - - if debug_mode: - print(target_file) - - client_command = _download_command( - command, - metadata["cls_path"], - metadata["cls_name"], - app_id, - debug_mode=debug_mode, - target_file=target_file if debug_mode else _resolve_command_path(command), - ) - client_command._setup(command_name=command, app_url=url) - sys.argv = sys.argv[len(command.split("_")) :] - client_command.run() diff --git a/src/lightning/app/cli/commands/cd.py b/src/lightning/app/cli/commands/cd.py deleted file mode 100644 index 7f84b894bf155..0000000000000 --- a/src/lightning/app/cli/commands/cd.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from typing import Optional, Tuple, Union - -import click -from rich.live import Live -from rich.spinner import Spinner -from rich.text import Text - -from lightning.app.cli.commands import ls -from lightning.app.cli.connect.app import _LIGHTNING_CONNECTION_FOLDER -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.cli_helpers import _error_and_exit - -logger = Logger(__name__) - -_HOME = os.path.expanduser("~") -_CD_FILE = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "cd.txt") - - -@click.argument("path", nargs=-1) -def cd(path: Optional[Union[Tuple[str], str]], verify: bool = True) -> None: - """Change the current directory within the Lightning Cloud filesystem.""" - with Live(Spinner("point", text=Text("pending...", style="white")), transient=True) as live: - root = "/" - - if isinstance(path, Tuple) and len(path) > 0: - path = " ".join(path) - - # handle ~/ - if isinstance(path, str) and path.startswith(_HOME): - path = "/" + path.replace(_HOME, "") - - # handle no path -> / - if path is None or len(path) == 0: - path = "/" - - if not os.path.exists(_LIGHTNING_CONNECTION_FOLDER): - os.makedirs(_LIGHTNING_CONNECTION_FOLDER) - - if not os.path.exists(_CD_FILE): - # Start from the root - if path.startswith(".."): - root = _apply_double_dots(root, path) - - with open(_CD_FILE, "w") as f: - f.write(root + "\n") - - live.stop() - - print(f"cd {root}") - - return root - - # read from saved cd - with open(_CD_FILE) as f: - lines = f.readlines() - root = lines[0].replace("\n", "") - - if verify: - if path.startswith("/"): - paths = [os.path.join(path, p) for p in ls.ls(path, print=False, use_live=False)] - else: - paths = [os.path.join(root, p) for p in ls.ls(root, print=False, use_live=False)] - - # generate new root - if root == "/": - if path == "/": - root = "/" - elif not path.startswith(".."): - if not path.startswith("/"): - path = "/" + path - root = path - else: - root = _apply_double_dots(root, path) - else: - if path.startswith(".."): - root = _apply_double_dots(root, path) - elif path.startswith("~"): - root = path[2:] - else: - root = os.path.join(root, path) - - if verify and root != "/" and not any(p.startswith(root) or root.startswith(p) for p in paths): - _error_and_exit(f"no such file or directory: {path}") - - os.remove(_CD_FILE) - - # store new root - with open(_CD_FILE, "w") as f: - f.write(root + "\n") - - live.stop() - - print(f"cd {root}") - - return root - - -def _apply_double_dots(root: str, path: str) -> str: - splits = [split for split in path.split("/") if split != ""] - for split in splits: - root = "/" + os.path.join(*root.split("/")[:-1]) if split == ".." else os.path.join(root, split) - return root diff --git a/src/lightning/app/cli/commands/cp.py b/src/lightning/app/cli/commands/cp.py deleted file mode 100644 index 0b11a874b216d..0000000000000 --- a/src/lightning/app/cli/commands/cp.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import concurrent -import contextlib -import os -import sys -from functools import partial -from multiprocessing.pool import ApplyResult -from pathlib import Path -from textwrap import dedent -from typing import Any, Optional, Tuple, Union - -import click -import requests -import urllib3 -from lightning_cloud.openapi import ( - Externalv1Cluster, - Externalv1LightningappInstance, - ProjectIdStorageBody, - V1CloudSpace, -) -from rich.live import Live -from rich.progress import BarColumn, DownloadColumn, Progress, TaskID, TextColumn -from rich.spinner import Spinner -from rich.text import Text - -from lightning.app.cli.commands.ls import _collect_artifacts, _get_prefix -from lightning.app.cli.commands.pwd import _pwd -from lightning.app.source_code import FileUploader -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.auth import _AuthTokenGetter -from lightning.app.utilities.cli_helpers import _error_and_exit -from lightning.app.utilities.network import LightningClient - -logger = Logger(__name__) - - -@click.argument("src_path", required=True) -@click.argument("dst_path", required=True) -@click.option("-r", required=False, hidden=True) -@click.option("--recursive", required=False, hidden=True) -@click.option("--zip", required=False, is_flag=True, default=False) -def cp(src_path: str, dst_path: str, r: bool = False, recursive: bool = False, zip: bool = False) -> None: - """Copy files between your local filesystem and the Lightning Cloud filesystem.""" - if sys.platform == "win32": - print("`cp` isn't supported on windows. Open an issue on Github.") - sys.exit(0) - - with Live(Spinner("point", text=Text("pending...", style="white")), transient=True) as live: - pwd = _pwd() - - client = LightningClient(retry=False) - - src_path, src_remote = _sanitize_path(src_path, pwd) - dst_path, dst_remote = _sanitize_path(dst_path, pwd) - - if src_remote and dst_remote: - return _error_and_exit("Moving files remotely isn't supported yet. Please, open a Github issue.") - - if not src_remote and dst_remote: - if dst_path == "/" or len(dst_path.split("/")) == 1: - return _error_and_exit("Uploading files at the project level isn't allowed yet.") - if zip: - return _error_and_exit("Zipping uploads isn't supported yet. Please, open a Github issue.") - _upload_files(live, client, src_path, dst_path, pwd) - return None - if src_remote and not dst_remote: - if zip: - return _zip_files(live, src_path, dst_path) - _download_files(live, client, src_path, dst_path, pwd) - return None - - return _error_and_exit("Moving files locally isn't supported yet. Please, open a Github issue.") - - -def _upload_files(live, client: LightningClient, local_src: str, remote_dst: str, pwd: str) -> str: - remote_splits = [split for split in remote_dst.split("/") if split != ""] - remote_dst = os.path.join(*remote_splits) - - if not os.path.exists(local_src): - return _error_and_exit(f"The provided source path {local_src} doesn't exist.") - - lit_resource = None - - if len(remote_splits) > 1: - project_id, lit_resource = _get_project_id_and_resource(pwd) - else: - project_id = _get_project_id_from_name(remote_dst) - - if len(remote_splits) > 2: - remote_dst = os.path.join(*remote_splits[2:]) - - local_src = Path(local_src).resolve() - upload_paths = [] - - if os.path.isdir(local_src): - for root_dir, _, paths in os.walk(local_src): - for path in paths: - upload_paths.append(os.path.join(root_dir, path)) - else: - upload_paths = [local_src] - - _upload_urls = [] - - clusters = client.projects_service_list_project_cluster_bindings(project_id) - - live.stop() - - for upload_path in upload_paths: - for cluster in clusters.clusters: - filename = str(upload_path).replace(str(os.getcwd()), "")[1:] - filename = _get_prefix(os.path.join(remote_dst, filename), lit_resource) if lit_resource else "/" + filename - - response = client.lightningapp_instance_service_upload_project_artifact( - project_id=project_id, - body=ProjectIdStorageBody(cluster_id=cluster.cluster_id, filename=filename), - async_req=True, - ) - _upload_urls.append(response) - - upload_urls = [] - for upload_url in _upload_urls: - upload_urls.extend(upload_url.get().urls) - - live.stop() - - if not upload_paths: - print("There were no files to upload.") - return None - - progress = _get_progress_bar() - - total_size = sum([Path(path).stat().st_size for path in upload_paths]) // max(len(clusters.clusters), 1) - task_id = progress.add_task("upload", filename="", total=total_size) - - progress.start() - - _upload_partial = partial(_upload, progress=progress, task_id=task_id) - - with concurrent.futures.ThreadPoolExecutor(4) as executor: - results = executor.map(_upload_partial, upload_paths, upload_urls) - - progress.stop() - - # Raise the first exception found - exception = next((e for e in results if isinstance(e, Exception)), None) - if exception: - _error_and_exit("We detected errors in uploading your files.") - return None - return None - - -def _upload(source_file: str, presigned_url: ApplyResult, progress: Progress, task_id: TaskID) -> Optional[Exception]: - source_file = Path(source_file) - file_uploader = FileUploader( - presigned_url, - source_file, - total_size=None, - name=str(source_file), - ) - file_uploader.progress = progress - file_uploader.task_id = task_id - file_uploader.upload() - - -def _zip_files(live: Live, remote_src: str, local_dst: str) -> None: - if len(remote_src.split("/")) < 3: - return _error_and_exit( - dedent( - f""" - The source path must be at least two levels deep (e.g. r:/my-project/my-lit-resource). - - The path provided was: r:{remote_src} - """ - ) - ) - - if os.path.isdir(local_dst): - local_dst = os.path.join(local_dst, os.path.basename(remote_src) + ".zip") - - project_id, lit_resource = _get_project_id_and_resource(remote_src) - - # /my-project/my-lit-resource/artfact-path -> cloudspace/my-lit-resource-id/artifact-path - artifact = "/".join(remote_src.split("/")[3:]) - prefix = _get_prefix(artifact, lit_resource) - - token = _AuthTokenGetter(LightningClient().api_client)._get_api_token() - endpoint = f"/v1/projects/{project_id}/artifacts/download?prefix={prefix}&token={token}" - - cluster = _cluster_from_lit_resource(lit_resource) - url = _storage_host(cluster) + endpoint - - live.stop() - progress = _get_progress_bar(transient=True) - progress.start() - task_id = progress.add_task("download zip", total=None) - - _download_file(local_dst, url, progress, task_id) - progress.stop() - - click.echo(f"Downloaded to {local_dst}") - return None - - -def _download_files(live, client, remote_src: str, local_dst: str, pwd: str): - project_id, lit_resource = _get_project_id_and_resource(pwd) - - download_paths = [] - download_urls = [] - total_size = [] - - prefix = _get_prefix("/".join(pwd.split("/")[3:]), lit_resource) + "/" - - for artifact in _collect_artifacts(client, project_id, prefix, include_download_url=True): - path = os.path.join(local_dst, artifact.filename.replace(remote_src, "")) - path = Path(path).resolve() - os.makedirs(path.parent, exist_ok=True) - download_paths.append(path) - download_urls.append(artifact.url) - total_size.append(int(artifact.size_bytes)) - - live.stop() - - if not download_paths: - print("There were no files to download.") - return - - progress = progress = _get_progress_bar() - - progress.start() - - task_id = progress.add_task("download", filename="", total=sum(total_size)) - - _download_file_fn = partial(_download_file, progress=progress, task_id=task_id) - - with concurrent.futures.ThreadPoolExecutor(4) as executor: - results = executor.map(_download_file_fn, download_paths, download_urls) - - progress.stop() - - # Raise the first exception found - exception = next((e for e in results if isinstance(e, Exception)), None) - if exception: - _error_and_exit("There was an error downloading your files.") - - -def _download_file(path: str, url: str, progress: Progress, task_id: TaskID) -> None: - # Disable warning about making an insecure request - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - with contextlib.suppress(ConnectionError): - request = requests.get(url, stream=True, verify=False) # noqa: S501 - - chunk_size = 1024 - - with open(path, "wb") as fp: - for chunk in request.iter_content(chunk_size=chunk_size): - fp.write(chunk) # type: ignore - progress.update(task_id, advance=len(chunk)) - - -def _sanitize_path(path: str, pwd: str) -> Tuple[str, bool]: - is_remote = _is_remote(path) - if is_remote: - path = _remove_remote(path) - path = pwd if path == "." else os.path.join(pwd, path) - return path, is_remote - - -def _is_remote(path: str) -> bool: - return path.startswith("r:") or path.startswith("remote:") - - -def _remove_remote(path: str) -> str: - return path.replace("r:", "").replace("remote:", "") - - -def _get_project_id_and_resource(pwd: str) -> Tuple[str, Union[Externalv1LightningappInstance, V1CloudSpace]]: - """Convert a root path to a project id and app id.""" - # TODO: Handle project level - project_name, resource_name, *_ = pwd.split("/")[1:3] - - # 1. Collect the projects of the user - client = LightningClient() - projects = client.projects_service_list_memberships() - project_id = [project.project_id for project in projects.memberships if project.name == project_name][0] - - # 2. Collect resources - lit_apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project_id).lightningapps - - lit_cloud_spaces = client.cloud_space_service_list_cloud_spaces(project_id=project_id).cloudspaces - - lit_ressources = [lit_resource for lit_resource in lit_cloud_spaces if lit_resource.name == resource_name] - - if len(lit_ressources) == 0: - lit_ressources = [lit_resource for lit_resource in lit_apps if lit_resource.name == resource_name] - - if len(lit_ressources) == 0: - print(f"ERROR: There isn't any Lightning Ressource matching the name {resource_name}.") - sys.exit(0) - - return project_id, lit_ressources[0] - - -def _get_project_id_from_name(project_name: str) -> str: - # 1. Collect the projects of the user - client = LightningClient() - projects = client.projects_service_list_memberships() - return [project.project_id for project in projects.memberships if project.name == project_name][0] - - -def _get_progress_bar(**kwargs: Any) -> Progress: - return Progress( - TextColumn("[bold blue]{task.description}", justify="left"), - BarColumn(bar_width=None), - "[self.progress.percentage]{task.percentage:>3.1f}%", - DownloadColumn(), - **kwargs, - ) - - -def _storage_host(cluster: Externalv1Cluster) -> str: - dev_host = os.environ.get("LIGHTNING_STORAGE_HOST") - if dev_host: - return dev_host - return f"https://storage.{cluster.spec.driver.kubernetes.root_domain_name}" - - -def _cluster_from_lit_resource(lit_resource: Union[Externalv1LightningappInstance, V1CloudSpace]) -> Externalv1Cluster: - client = LightningClient() - if isinstance(lit_resource, Externalv1LightningappInstance): - return client.cluster_service_get_cluster(lit_resource.spec.cluster_id) - - clusters = client.cluster_service_list_clusters() - for cluster in clusters.clusters: - if cluster.id == clusters.default_cluster: - return cluster - return None diff --git a/src/lightning/app/cli/commands/logs.py b/src/lightning/app/cli/commands/logs.py deleted file mode 100644 index 4587987ae5f17..0000000000000 --- a/src/lightning/app/cli/commands/logs.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import click -import rich -from rich.color import ANSI_COLOR_NAMES - -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.app_logs import _app_logs_reader -from lightning.app.utilities.cloud import _get_project -from lightning.app.utilities.logs_socket_api import _LightningLogsSocketAPI -from lightning.app.utilities.network import LightningClient - -logger = Logger(__name__) - - -@click.argument("app_name", required=False) -@click.argument("components", nargs=-1, required=False) -@click.option("-f", "--follow", required=False, is_flag=True, help="Wait for new logs, to exit use CTRL+C.") -def logs(app_name: str, components: List[str], follow: bool) -> None: - """Show cloud application logs. By default, prints logs for all currently available components. - - Example uses: - - Print all application logs: - - $ lightning show logs my-application - - Print logs only from the flow (no work): - - $ lightning show logs my-application flow - - Print logs only from selected works: - - $ lightning show logs my-application root.work_a root.work_b - - """ - _show_logs(app_name, components, follow) - - -def _show_logs(app_name: str, components: List[str], follow: bool) -> None: - client = LightningClient(retry=False) - project = _get_project(client) - - apps = { - getattr(app, "display_name", None) or app.name: app - for app in client.lightningapp_instance_service_list_lightningapp_instances( - project_id=project.project_id - ).lightningapps - } - - if not apps: - raise click.ClickException( - "You don't have any application in the cloud. Please, run an application first with `--cloud`." - ) - - if not app_name: - raise click.ClickException( - f"You have not specified any Lightning App. Please select one of the following: [{', '.join(apps.keys())}]." - ) - - if app_name not in apps: - raise click.ClickException( - f"The Lightning App '{app_name}' does not exist. " - f"Please select one of the following: [{', '.join(apps.keys())}]." - ) - - # Fetch all lightning works from given application - # 'Flow' component is somewhat implicit, only one for whole app, - # and not listed in lightningwork API - so we add it directly to the list - works = client.lightningwork_service_list_lightningwork( - project_id=project.project_id, app_id=apps[app_name].id - ).lightningworks - - app_component_names = ["flow"] + [f.name for f in apps[app_name].spec.flow_servers] + [w.name for w in works] - - if not components: - components = app_component_names - - else: - - def add_prefix(c: str) -> str: - if c == "flow": - return c - if not c.startswith("root."): - return "root." + c - return c - - components = [add_prefix(c) for c in components] - - for component in components: - if component not in app_component_names: - raise click.ClickException(f"Component '{component}' does not exist in app {app_name}.") - - log_reader = _app_logs_reader( - logs_api_client=_LightningLogsSocketAPI(client.api_client), - project_id=project.project_id, - app_id=apps[app_name].id, - component_names=components, - follow=follow, - ) - - rich_colors = list(ANSI_COLOR_NAMES) - colors = {c: rich_colors[i + 1] for i, c in enumerate(components)} - - for log_event in log_reader: - date = log_event.timestamp.strftime("%m/%d/%Y %H:%M:%S") - color = colors[log_event.component_name] - rich.print(f"[{color}]{log_event.component_name}[/{color}] {date} {log_event.message}") diff --git a/src/lightning/app/cli/commands/ls.py b/src/lightning/app/cli/commands/ls.py deleted file mode 100644 index e16a354f66d8c..0000000000000 --- a/src/lightning/app/cli/commands/ls.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import contextlib -import os -import sys -from contextlib import nullcontext -from typing import Generator, List, Optional - -import click -import lightning_cloud -import rich -from lightning_cloud.openapi import Externalv1LightningappInstance -from rich.console import Console -from rich.live import Live -from rich.spinner import Spinner -from rich.text import Text - -from lightning.app.cli.connect.app import _LIGHTNING_CONNECTION_FOLDER -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.cli_helpers import _error_and_exit -from lightning.app.utilities.network import LightningClient - -_FOLDER_COLOR = "sky_blue1" -_FILE_COLOR = "white" - -logger = Logger(__name__) - - -@click.argument("path", required=False) -def ls(path: Optional[str] = None, print: bool = True, use_live: bool = True) -> List[str]: - """List the contents of a folder in the Lightning Cloud Filesystem.""" - from lightning.app.cli.commands.cd import _CD_FILE - - if sys.platform == "win32": - _error_and_exit("`ls` isn't supported on windows. Open an issue on Github.") - - root = "/" - - context = ( - Live(Spinner("point", text=Text("pending...", style="white")), transient=True) if use_live else nullcontext() - ) - - with context: - if not os.path.exists(_LIGHTNING_CONNECTION_FOLDER): - os.makedirs(_LIGHTNING_CONNECTION_FOLDER) - - if not os.path.exists(_CD_FILE): - with open(_CD_FILE, "w") as f: - f.write(root + "\n") - else: - with open(_CD_FILE) as f: - lines = f.readlines() - root = lines[0].replace("\n", "") - - client = LightningClient(retry=False) - projects = client.projects_service_list_memberships() - - if root == "/": - project_names = [project.name for project in projects.memberships] - if print: - _print_names_with_colors(project_names, [_FOLDER_COLOR] * len(project_names)) - return project_names - - # Note: Root format has the following structure: - # /{PROJECT_NAME}/{APP_NAME}/{ARTIFACTS_PATHS} - splits = root.split("/")[1:] - - project = [project for project in projects.memberships if project.name == splits[0]] - - # This happens if the user changes cluster and the project doesn't exit. - if len(project) == 0: - return _error_and_exit( - f"There isn't any Lightning Project matching the name {splits[0]}." " HINT: Use `lightning_app cd`." - ) - - project_id = project[0].project_id - - # Parallelise calls - lit_apps = client.lightningapp_instance_service_list_lightningapp_instances( - project_id=project_id, async_req=True - ) - lit_cloud_spaces = client.cloud_space_service_list_cloud_spaces(project_id=project_id, async_req=True) - - lit_apps = lit_apps.get().lightningapps - lit_cloud_spaces = lit_cloud_spaces.get().cloudspaces - - if len(splits) == 1: - apps = [lit_app.name for lit_app in lit_apps] - cloud_spaces = [lit_cloud_space.name for lit_cloud_space in lit_cloud_spaces] - ressource_names = sorted(set(cloud_spaces + apps)) - if print: - _print_names_with_colors(ressource_names, [_FOLDER_COLOR] * len(ressource_names)) - return ressource_names - - lit_ressources = [lit_resource for lit_resource in lit_cloud_spaces if lit_resource.name == splits[1]] - - if len(lit_ressources) == 0: - lit_ressources = [lit_resource for lit_resource in lit_apps if lit_resource.name == splits[1]] - - if len(lit_ressources) == 0: - _error_and_exit(f"There isn't any Lightning Ressource matching the name {splits[1]}.") - - lit_resource = lit_ressources[0] - - app_paths = [] - app_colors = [] - - cloud_spaces_paths = [] - cloud_spaces_colors = [] - - depth = len(splits) - - prefix = "/".join(splits[2:]) - prefix = _get_prefix(prefix, lit_resource) - - for artifact in _collect_artifacts(client=client, project_id=project_id, prefix=prefix): - if str(artifact.filename).startswith("/"): - artifact.filename = artifact.filename[1:] - - path = os.path.join(project_id, prefix[1:], artifact.filename) - - artifact_splits = path.split("/") - - if len(artifact_splits) <= depth + 1: - continue - - path = artifact_splits[depth + 1] - - paths = app_paths if isinstance(lit_resource, Externalv1LightningappInstance) else cloud_spaces_paths - colors = app_colors if isinstance(lit_resource, Externalv1LightningappInstance) else cloud_spaces_colors - - if path not in paths: - paths.append(path) - - # display files otherwise folders - colors.append(_FILE_COLOR if len(artifact_splits) == depth + 1 else _FOLDER_COLOR) - - if print: - if app_paths and cloud_spaces_paths: - if app_paths: - rich.print("Lightning App") - _print_names_with_colors(app_paths, app_colors) - - if cloud_spaces_paths: - rich.print("Lightning CloudSpaces") - _print_names_with_colors(cloud_spaces_paths, cloud_spaces_colors) - else: - _print_names_with_colors(app_paths + cloud_spaces_paths, app_colors + cloud_spaces_colors) - - return app_paths + cloud_spaces_paths - - -def _add_colors(filename: str, color: Optional[str] = None) -> str: - return f"[{color}]{filename}[/{color}]" - - -def _print_names_with_colors(names: List[str], colors: List[str], padding: int = 5) -> None: - console = Console() - width = console.width - - max_L = max([len(name) for name in names] + [0]) + padding - - use_spacing = False - - if max_L * len(names) < width: - use_spacing = True - - num_cols = width // max_L - - columns = {} - for index, (name, color) in enumerate(zip(names, colors)): - row = index // num_cols - if row not in columns: - columns[row] = [] - columns[row].append((name, color)) - - for row_index in sorted(columns): - row = "" - for name, color in columns[row_index]: - spacing = padding if use_spacing else max_L - len(name) - spaces = " " * spacing - row += _add_colors(name, color) + spaces - rich.print(row) - - -def _collect_artifacts( - client: LightningClient, - project_id: str, - prefix: str = "", - page_token: Optional[str] = "", - cluster_id: Optional[str] = None, - page_size: int = 100_000, - tokens=None, - include_download_url: bool = False, -) -> Generator: - if tokens is None: - tokens = [] - - if cluster_id is None: - clusters = client.projects_service_list_project_cluster_bindings(project_id) - for cluster in clusters.clusters: - yield from _collect_artifacts( - client, - project_id, - prefix=prefix, - cluster_id=cluster.cluster_id, - page_token=page_token, - tokens=tokens, - page_size=page_size, - include_download_url=include_download_url, - ) - else: - if page_token in tokens: - return - - # Note: This is triggered when the request is wrong. - # This is currently happening due to looping through the user clusters. - with contextlib.suppress(lightning_cloud.openapi.rest.ApiException): - response = client.lightningapp_instance_service_list_project_artifacts( - project_id, - prefix=prefix, - cluster_id=cluster_id, - page_token=page_token, - include_download_url=include_download_url, - page_size=str(page_size), - ) - for artifact in response.artifacts: - if ".lightning-app-sync" in artifact.filename: - continue - yield artifact - - if response.next_page_token: - tokens.append(page_token) - yield from _collect_artifacts( - client, - project_id, - prefix=prefix, - cluster_id=cluster_id, - page_token=response.next_page_token, - tokens=tokens, - ) - - -def _add_resource_prefix(prefix: str, resource_path: str): - if resource_path in prefix: - return prefix - prefix = os.path.join(resource_path, prefix) - if not prefix.startswith("/"): - prefix = "/" + prefix - return prefix - - -def _get_prefix(prefix: str, lit_resource) -> str: - if isinstance(lit_resource, Externalv1LightningappInstance): - return _add_resource_prefix(prefix, f"lightningapps/{lit_resource.id}") - - return _add_resource_prefix(prefix, f"cloudspaces/{lit_resource.id}") diff --git a/src/lightning/app/cli/commands/pwd.py b/src/lightning/app/cli/commands/pwd.py deleted file mode 100644 index 7768309e4e6bb..0000000000000 --- a/src/lightning/app/cli/commands/pwd.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -from rich.live import Live -from rich.spinner import Spinner -from rich.text import Text - -from lightning.app.cli.commands.cd import _CD_FILE -from lightning.app.utilities.app_helpers import Logger - -logger = Logger(__name__) - - -def pwd() -> str: - """Print your current working directory in the Lightning Cloud filesystem.""" - if sys.platform == "win32": - print("`pwd` isn't supported on windows. Open an issue on Github.") - sys.exit(0) - - with Live(Spinner("point", text=Text("pending...", style="white")), transient=True): - root = _pwd() - - print(root) - - return root - - -def _pwd() -> str: - root = "/" - - if not os.path.exists(_CD_FILE): - with open(_CD_FILE, "w") as f: - f.write(root + "\n") - else: - with open(_CD_FILE) as f: - lines = f.readlines() - root = lines[0].replace("\n", "") - - return root diff --git a/src/lightning/app/cli/commands/rm.py b/src/lightning/app/cli/commands/rm.py deleted file mode 100644 index 587cc50469131..0000000000000 --- a/src/lightning/app/cli/commands/rm.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import contextlib -import os - -import click -import lightning_cloud -import rich - -from lightning.app.cli.commands.ls import _add_colors, _get_prefix -from lightning.app.cli.commands.pwd import _pwd -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.cli_helpers import _error_and_exit -from lightning.app.utilities.network import LightningClient - -logger = Logger(__name__) - - -@click.argument("rm_path", required=True) -@click.option("-r", required=False, hidden=True) -@click.option("--recursive", required=False, hidden=True) -def rm(rm_path: str, r: bool = False, recursive: bool = False) -> None: - """Delete files on the Lightning Cloud filesystem.""" - root = _pwd() - - if rm_path in (".", ".."): - return _error_and_exit('rm "." and ".." may not be removed') - - if ".." in rm_path: - return _error_and_exit('rm ".." or higher may not be removed') - - root = os.path.join(root, rm_path) - splits = [split for split in root.split("/") if split != ""] - - if root == "/" or len(splits) == 1: - return _error_and_exit("rm at the project level isn't supported") - - client = LightningClient(retry=False) - projects = client.projects_service_list_memberships() - - project = [project for project in projects.memberships if project.name == splits[0]] - - # This happens if the user changes cluster and the project doesn't exist. - if len(project) == 0: - return _error_and_exit( - f"There isn't any Lightning Project matching the name {splits[0]}." " HINT: Use `lightning cd`." - ) - - project_id = project[0].project_id - - # Parallelise calls - lit_apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project_id, async_req=True) - lit_cloud_spaces = client.cloud_space_service_list_cloud_spaces(project_id=project_id, async_req=True) - - lit_apps = lit_apps.get().lightningapps - lit_cloud_spaces = lit_cloud_spaces.get().cloudspaces - - lit_ressources = [lit_resource for lit_resource in lit_cloud_spaces if lit_resource.name == splits[1]] - - if len(lit_ressources) == 0: - lit_ressources = [lit_resource for lit_resource in lit_apps if lit_resource.name == splits[1]] - - if len(lit_ressources) == 0: - _error_and_exit(f"There isn't any Lightning Ressource matching the name {splits[1]}.") - - lit_resource = lit_ressources[0] - - prefix = "/".join(splits[2:]) - prefix = _get_prefix(prefix, lit_resource) - - clusters = client.projects_service_list_project_cluster_bindings(project_id) - succeeded = False - - for cluster in clusters.clusters: - with contextlib.suppress(lightning_cloud.openapi.rest.ApiException): - client.lightningapp_instance_service_delete_project_artifact( - project_id=project_id, - cluster_id=cluster.cluster_id, - filename=prefix, - ) - succeeded = True - break - - prefix = os.path.join(*splits) - - if succeeded: - rich.print(_add_colors(f"Successfuly deleted `{prefix}`.", color="green")) - return None - - return _error_and_exit(f"No file or folder named `{prefix}` was found.") diff --git a/src/lightning/app/cli/component-template/.github/workflows/ci-testing.yml b/src/lightning/app/cli/component-template/.github/workflows/ci-testing.yml deleted file mode 100644 index 16abb8f418b89..0000000000000 --- a/src/lightning/app/cli/component-template/.github/workflows/ci-testing.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: CI testing - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - # Trigger the workflow on push or pull request, but only for the master branch - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - pytest: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-20.04, macOS-11, windows-2019] - python-version: [3.8] - - # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 35 - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646 - - name: Setup macOS - if: runner.os == 'macOS' - run: | - brew install libomp # https://github.com/pytorch/pytorch/issues/20030 - - - name: Get pip cache dir - id: pip-cache - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-py${{ matrix.python-version }}- - - - name: Clone Template React UI Repo - uses: actions/checkout@v3 - with: - repository: Lightning-AI/lightning - token: ${{ secrets.PAT_GHOST }} - ref: "master" - path: lightning - - - name: Install Lightning - run: | - cd lightning - pip install -r requirements.txt - pip install -e . - shell: bash - - - name: Install dependencies - run: | - python --version - pip --version - pip install --requirement requirements.txt --upgrade --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - pip install --requirement tests/requirements.txt --quiet - pip list - shell: bash - - - name: Tests - run: | - coverage run --source placeholdername -m py.test placeholdername tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml - - - name: Statistics - if: success() - run: | - coverage report diff --git a/src/lightning/app/cli/component-template/.gitignore b/src/lightning/app/cli/component-template/.gitignore deleted file mode 100644 index 70ba25888435f..0000000000000 --- a/src/lightning/app/cli/component-template/.gitignore +++ /dev/null @@ -1,157 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -*install-app* - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg - -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Sphinx documentation -docs/_build/ -docs/source/api/ -docs/source/*.md - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.local_env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# PyCharm -.idea/ - -# Lightning logs -lightning_logs -*.gz -.DS_Store -.*_submit.py -.vscode - -MNIST -*.pt -.storage/ -.shared/ -infra -data -coverage.* -# Frontend build artifacts -*lightning/app/ui* -gradio_cached_examples -/docs/source/api_reference/generated/* -examples/my_own_leaderboard/submissions/* -docs/source/api_reference/generated/* -*.ckpt -redis-stable -node_modules -*.rdb -*.webm -*hars -examples/quick_start/* -examples/quick_start -examples/template_react_ui/* -examples/template_react_ui -# Ignore external components -lightning/app/components/* -!lightning/app/components/python -!lightning/app/components/serve -!lightning/app/components/__init__.py -!lightning/app/components/README.md -train_script.py -*return_values* -scratch -storage diff --git a/src/lightning/app/cli/component-template/LICENSE b/src/lightning/app/cli/component-template/LICENSE deleted file mode 100644 index 261eeb9e9f8b2..0000000000000 --- a/src/lightning/app/cli/component-template/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/lightning/app/cli/component-template/README.md b/src/lightning/app/cli/component-template/README.md deleted file mode 100644 index 1d700e286461b..0000000000000 --- a/src/lightning/app/cli/component-template/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# placeholdername component - -This ⚡ [Lightning component](https://lightning.ai/) ⚡ was generated automatically with: - -```bash -lightning_app init component placeholdername -``` - -## To run placeholdername - -First, install placeholdername (warning: this component has not been officially approved on the lightning gallery): - -```bash -lightning_app install component https://github.com/theUser/placeholdername -``` - -Once the app is installed, use it in an app: - -```python -from placeholdername import TemplateComponent -import lightning as L - - -class LitApp(L.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.placeholdername = TemplateComponent() - - def run(self): - print("this is a simple Lightning app to verify your component is working as expected") - self.placeholdername.run() - - -app = L.LightningApp(LitApp()) -``` diff --git a/src/lightning/app/cli/component-template/app.py b/src/lightning/app/cli/component-template/app.py deleted file mode 100644 index 0a10532204043..0000000000000 --- a/src/lightning/app/cli/component-template/app.py +++ /dev/null @@ -1,15 +0,0 @@ -from lightning.app import LightningApp, LightningFlow -from placeholdername import TemplateComponent - - -class LitApp(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.placeholdername = TemplateComponent() - - def run(self): - print("this is a simple Lightning app to verify your component is working as expected") - self.placeholdername.run() - - -app = LightningApp(LitApp()) diff --git a/src/lightning/app/cli/component-template/placeholdername/__init__.py b/src/lightning/app/cli/component-template/placeholdername/__init__.py deleted file mode 100644 index 92b4ef47d8062..0000000000000 --- a/src/lightning/app/cli/component-template/placeholdername/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from placeholdername.component import TemplateComponent - -__all__ = ["TemplateComponent"] diff --git a/src/lightning/app/cli/component-template/placeholdername/component.py b/src/lightning/app/cli/component-template/placeholdername/component.py deleted file mode 100644 index 251a4e10c6a9f..0000000000000 --- a/src/lightning/app/cli/component-template/placeholdername/component.py +++ /dev/null @@ -1,12 +0,0 @@ -from lightning.app import LightningWork - - -class TemplateComponent(LightningWork): - def __init__(self) -> None: - super().__init__() - self.value = 0 - - def run(self): - self.value += 1 - print("welcome to your work component") - print("this is running inside a work") diff --git a/src/lightning/app/cli/component-template/requirements.txt b/src/lightning/app/cli/component-template/requirements.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/component-template/setup.py b/src/lightning/app/cli/component-template/setup.py deleted file mode 100644 index 78631901190b2..0000000000000 --- a/src/lightning/app/cli/component-template/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python - -from setuptools import find_packages, setup - -setup( - name="placeholdername", - version="0.0.0", - description="⚡ Lightning component ⚡ generated with command: lightning_app init component", - author="", - author_email="", - # REPLACE WITH YOUR OWN GITHUB PROJECT LINK - url="https://github.com/Lightning-AI/lightning-component-template", - install_requires=[], - packages=find_packages(), -) diff --git a/src/lightning/app/cli/component-template/tests/README.md b/src/lightning/app/cli/component-template/tests/README.md deleted file mode 100644 index bef681691185d..0000000000000 --- a/src/lightning/app/cli/component-template/tests/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Run tests - -To run the tests: - -```bash -# go to your component folder -cd placeholdername - -# go to tests folder -cd tests - -# install testing deps -pip install -r requirements.txt - -# run tests -pytest . -``` diff --git a/src/lightning/app/cli/component-template/tests/__init__.py b/src/lightning/app/cli/component-template/tests/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/component-template/tests/requirements.txt b/src/lightning/app/cli/component-template/tests/requirements.txt deleted file mode 100644 index 3185d1c44f033..0000000000000 --- a/src/lightning/app/cli/component-template/tests/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -coverage -codecov>=2.1 -pytest>=5.0.0 -pytest-cov -pytest-flake8 -flake8 -check-manifest -twine==4.0.1 diff --git a/src/lightning/app/cli/component-template/tests/test_placeholdername_component.py b/src/lightning/app/cli/component-template/tests/test_placeholdername_component.py deleted file mode 100644 index 6b9c28845749c..0000000000000 --- a/src/lightning/app/cli/component-template/tests/test_placeholdername_component.py +++ /dev/null @@ -1,14 +0,0 @@ -r"""To test a lightning component: - -1. Init the component. -2. call .run() - -""" - -from placeholdername.component import TemplateComponent - - -def test_placeholder_component(): - messenger = TemplateComponent() - messenger.run() - assert messenger.value == 1 diff --git a/src/lightning/app/cli/connect/__init__.py b/src/lightning/app/cli/connect/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/connect/app.py b/src/lightning/app/cli/connect/app.py deleted file mode 100644 index e3f0d0b151bb5..0000000000000 --- a/src/lightning/app/cli/connect/app.py +++ /dev/null @@ -1,387 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import shutil -import sys -from subprocess import Popen -from typing import List, Optional, Tuple - -import click -import psutil -from lightning_utilities.core.imports import package_available -from rich.progress import Progress - -from lightning.app.utilities.cli_helpers import _get_app_display_name, _LightningAppOpenAPIRetriever -from lightning.app.utilities.cloud import _get_project -from lightning.app.utilities.enum import OpenAPITags -from lightning.app.utilities.log import get_logfile -from lightning.app.utilities.network import LightningClient - -_HOME = os.path.expanduser("~") -_PPID = os.getenv("LIGHTNING_CONNECT_PPID", str(psutil.Process(os.getpid()).ppid())) -_LIGHTNING_CONNECTION = os.path.join(_HOME, ".lightning", "lightning_connection") -_LIGHTNING_CONNECTION_FOLDER = os.path.join(_LIGHTNING_CONNECTION, _PPID) - - -@click.argument("app_name_or_id", required=True) -def connect_app(app_name_or_id: str): - """Connect your local terminal to a running lightning app. - - After connecting, the lightning CLI will respond to commands exposed by the app. - - Example: - - \b - # connect to an app named pizza-cooker-123 - lightning connect pizza-cooker-123 - \b - # this will now show the commands exposed by pizza-cooker-123 - lightning --help - \b - # while connected, you can run the cook-pizza command exposed - # by pizza-cooker-123.BTW, this should arguably generate an exception :-) - lightning cook-pizza --flavor pineapple - \b - # once done, disconnect and go back to the standard lightning CLI commands - lightning disconnect - - """ - from lightning.app.utilities.commands.base import _download_command - - _clean_lightning_connection() - - if not os.path.exists(_LIGHTNING_CONNECTION_FOLDER): - os.makedirs(_LIGHTNING_CONNECTION_FOLDER) - - connected_file = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "connect.txt") - - matched_connection_path = _scan_lightning_connections(app_name_or_id) - - if os.path.exists(connected_file): - with open(connected_file) as f: - result = f.readlines()[0].replace("\n", "") - - if result == app_name_or_id: - if app_name_or_id == "localhost": - click.echo("You are connected to the local Lightning App.") - else: - click.echo(f"You are already connected to the cloud Lightning App: {app_name_or_id}.") - else: - disconnect_app() - connect_app(app_name_or_id) - - elif app_name_or_id.startswith("localhost"): - with Progress() as progress_bar: - connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=1.0) - - if app_name_or_id != "localhost": - raise Exception("You need to pass localhost to connect to the local Lightning App.") - - retriever = _LightningAppOpenAPIRetriever(None) - - if retriever.api_commands is None: - raise Exception(f"Connection wasn't successful. Is your app {app_name_or_id} running?") - - increment = 1 / (1 + len(retriever.api_commands)) - - progress_bar.update(connecting, advance=increment) - - commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - if not os.path.exists(commands_folder): - os.makedirs(commands_folder) - - _write_commands_metadata(retriever.api_commands) - - with open(os.path.join(commands_folder, "openapi.json"), "w") as f: - json.dump(retriever.openapi, f) - - _install_missing_requirements(retriever) - - for command_name, metadata in retriever.api_commands.items(): - if "cls_path" in metadata: - target_file = os.path.join(commands_folder, f"{command_name.replace(' ', '_')}.py") - _download_command( - command_name, - metadata["cls_path"], - metadata["cls_name"], - None, - target_file=target_file, - ) - else: - with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: - f.write(command_name) - - progress_bar.update(connecting, advance=increment) - - with open(connected_file, "w") as f: - f.write(app_name_or_id + "\n") - - click.echo("The lightning App CLI now responds to app commands. Use 'lightning_app --help' to see them.") - click.echo(" ") - - Popen( - f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning_app --help", - shell=True, - stdout=sys.stdout, - stderr=sys.stderr, - ).wait() - - elif matched_connection_path: - matched_connected_file = os.path.join(matched_connection_path, "connect.txt") - matched_commands = os.path.join(matched_connection_path, "commands") - if os.path.isdir(matched_commands): - commands = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - shutil.copytree(matched_commands, commands) - shutil.copy(matched_connected_file, connected_file) - - click.echo("The lightning App CLI now responds to app commands. Use 'lightning_app --help' to see them.") - click.echo(" ") - - Popen( - f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning_app --help", - shell=True, - stdout=sys.stdout, - stderr=sys.stderr, - ).wait() - - else: - with Progress() as progress_bar: - connecting = progress_bar.add_task("[magenta]Setting things up for you...", total=1.0) - - retriever = _LightningAppOpenAPIRetriever(app_name_or_id) - - if not retriever.api_commands: - client = LightningClient(retry=False) - project = _get_project(client) - apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project.project_id) - click.echo( - "We didn't find a matching App. Here are the available Apps that you can " - f"connect to {[_get_app_display_name(app) for app in apps.lightningapps]}." - ) - return - - increment = 1 / (1 + len(retriever.api_commands)) - - progress_bar.update(connecting, advance=increment) - - _install_missing_requirements(retriever) - - commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - if not os.path.exists(commands_folder): - os.makedirs(commands_folder) - - _write_commands_metadata(retriever.api_commands) - - for command_name, metadata in retriever.api_commands.items(): - if "cls_path" in metadata: - target_file = os.path.join(commands_folder, f"{command_name}.py") - _download_command( - command_name, - metadata["cls_path"], - metadata["cls_name"], - retriever.app_id, - target_file=target_file, - ) - else: - with open(os.path.join(commands_folder, f"{command_name}.txt"), "w") as f: - f.write(command_name) - - progress_bar.update(connecting, advance=increment) - - with open(connected_file, "w") as f: - f.write(retriever.app_name + "\n") - f.write(retriever.app_id + "\n") - - click.echo("The lightning App CLI now responds to app commands. Use 'lightning_app --help' to see them.") - click.echo(" ") - - Popen( - f"LIGHTNING_CONNECT_PPID={_PPID} {sys.executable} -m lightning_app --help", - shell=True, - stdout=sys.stdout, - stderr=sys.stderr, - ).wait() - - -def disconnect_app(logout: bool = False): - """Disconnect from an App.""" - _clean_lightning_connection() - - connected_file = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "connect.txt") - if os.path.exists(connected_file): - with open(connected_file) as f: - result = f.readlines()[0].replace("\n", "") - - os.remove(connected_file) - commands_folder = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - if os.path.exists(commands_folder): - shutil.rmtree(commands_folder) - - if result == "localhost": - click.echo("You are disconnected from the local Lightning App.") - else: - click.echo(f"You are disconnected from the cloud Lightning App: {result}.") - else: - if not logout: - click.echo( - "You aren't connected to any Lightning App. " - "Please use `lightning_app connect app_name_or_id` to connect to one." - ) - - -def _read_connected_file(connected_file): - if os.path.exists(connected_file): - with open(connected_file) as f: - lines = [line.replace("\n", "") for line in f.readlines()] - if len(lines) == 2: - return lines[0], lines[1] - return lines[0], None - return None, None - - -def _retrieve_connection_to_an_app() -> Tuple[Optional[str], Optional[str]]: - connected_file = os.path.join(_LIGHTNING_CONNECTION_FOLDER, "connect.txt") - return _read_connected_file(connected_file) - - -def _get_commands_folder() -> str: - return os.path.join(_LIGHTNING_CONNECTION_FOLDER, "commands") - - -def _write_commands_metadata(api_commands): - metadata = dict(api_commands.items()) - metadata_path = os.path.join(_get_commands_folder(), ".meta.json") - with open(metadata_path, "w") as f: - json.dump(metadata, f) - - -def _get_commands_metadata(): - metadata_path = os.path.join(_get_commands_folder(), ".meta.json") - with open(metadata_path) as f: - return json.load(f) - - -def _resolve_command_path(command: str) -> str: - return os.path.join(_get_commands_folder(), f"{command}.py") - - -def _list_app_commands(echo: bool = True) -> List[str]: - metadata = _get_commands_metadata() - metadata = {key.replace("_", " "): value for key, value in metadata.items()} - - command_names = sorted(metadata.keys()) - if not command_names: - click.echo("The current Lightning App doesn't have commands.") - return [] - - app_info = metadata[command_names[0]].get("app_info", None) - - title, description, on_connect_end = "Lightning", None, None - if app_info: - title = app_info.get("title") - description = app_info.get("description") - on_connect_end = app_info.get("on_connect_end") - - if echo: - click.echo(f"{title} App") - if description: - click.echo("") - click.echo("Description:") - if description.endswith("\n"): - description = description[:-2] - click.echo(f" {description}") - click.echo("") - click.echo("Commands:") - max_length = max(len(n) for n in command_names) - for command_name in command_names: - padding = (max_length + 1 - len(command_name)) * " " - click.echo(f" {command_name}{padding}{metadata[command_name].get('description', '')}") - if "LIGHTNING_CONNECT_PPID" in os.environ and on_connect_end: - if on_connect_end.endswith("\n"): - on_connect_end = on_connect_end[:-2] - click.echo(on_connect_end) - return command_names - - -def _install_missing_requirements( - retriever: _LightningAppOpenAPIRetriever, - fail_if_missing: bool = False, -): - requirements = set() - for metadata in retriever.api_commands.values(): - if metadata["tag"] == OpenAPITags.APP_CLIENT_COMMAND: - for req in metadata.get("requirements", []) or []: - requirements.add(req) - - if requirements: - missing_requirements = [] - for req in requirements: - if not (package_available(req) or package_available(req.replace("-", "_"))): - missing_requirements.append(req) - - if missing_requirements: - if fail_if_missing: - missing_requirements = " ".join(missing_requirements) - print(f"The command failed as you are missing the following requirements: `{missing_requirements}`.") - sys.exit(0) - - for req in missing_requirements: - std_out_out = get_logfile("output.log") - with open(std_out_out, "wb") as stdout: - Popen( - f"{sys.executable} -m pip install {req}", - shell=True, - stdout=stdout, - stderr=stdout, - ).wait() - os.remove(std_out_out) - - -def _clean_lightning_connection(): - if not os.path.exists(_LIGHTNING_CONNECTION): - return - - for ppid in os.listdir(_LIGHTNING_CONNECTION): - try: - psutil.Process(int(ppid)) - except (psutil.NoSuchProcess, ValueError): - connection = os.path.join(_LIGHTNING_CONNECTION, str(ppid)) - if os.path.exists(connection): - shutil.rmtree(connection) - - -def _scan_lightning_connections(app_name_or_id): - if not os.path.exists(_LIGHTNING_CONNECTION): - return None - - for ppid in os.listdir(_LIGHTNING_CONNECTION): - try: - psutil.Process(int(ppid)) - except (psutil.NoSuchProcess, ValueError): - continue - - connection_path = os.path.join(_LIGHTNING_CONNECTION, str(ppid)) - - connected_file = os.path.join(connection_path, "connect.txt") - curr_app_name, curr_app_id = _read_connected_file(connected_file) - - if not curr_app_name: - continue - - if app_name_or_id in (curr_app_name, curr_app_id): - return connection_path - - return None diff --git a/src/lightning/app/cli/connect/data.py b/src/lightning/app/cli/connect/data.py deleted file mode 100644 index 6069432ddb187..0000000000000 --- a/src/lightning/app/cli/connect/data.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright The Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ast -import sys - -import click -import lightning_cloud -import rich -from rich.live import Live -from rich.spinner import Spinner -from rich.text import Text - -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.cli_helpers import _error_and_exit -from lightning.app.utilities.cloud import _get_project -from lightning.app.utilities.network import LightningClient - -logger = Logger(__name__) - - -@click.argument("name", required=True) -@click.option("--region", help="The AWS region of your bucket. Example: `us-west-1`.", required=True) -@click.option( - "--source", help="The URL path to your AWS S3 folder. Example: `s3://pl-flash-data/images/`.", required=True -) -@click.option( - "--secret_arn_name", - help="The name of role stored as a secret on Lightning AI to access your data. " - "Learn more with https://gist.github.com/tchaton/12ad4b788012e83c0eb35e6223ae09fc. " - "Example: `my_role`.", - required=False, -) -@click.option( - "--destination", help="Where your data should appear in the cloud. Currently not supported.", required=False -) -@click.option("--project_name", help="The project name on which to create the data connection.", required=False) -def connect_data( - name: str, - region: str, - source: str, - secret_arn_name: str = "", - destination: str = "", - project_name: str = "", -) -> None: - """Create a new data connection.""" - - from lightning_cloud.openapi import Create, V1AwsDataConnection - - if sys.platform == "win32": - _error_and_exit("Data connection isn't supported on windows. Open an issue on Github.") - - with Live(Spinner("point", text=Text("pending...", style="white")), transient=True) as live: - live.stop() - - client = LightningClient(retry=False) - projects = client.projects_service_list_memberships() - - project_id = None - - for project in projects.memberships: - if project.name == project_name: - project_id = project.project_id - break - - if project_id is None: - project_id = _get_project(client).project_id - - if not source.startswith("s3://"): - return _error_and_exit( - "Only public S3 folders are supported for now. Please, open a Github issue with your use case." - ) - - try: - client.data_connection_service_create_data_connection( - body=Create( - name=name, - aws=V1AwsDataConnection( - region=region, - source=source, - destination=destination, - secret_arn_name=secret_arn_name, - ), - ), - project_id=project_id, - ) - - # Note: Expose through lightning show data {DATA_NAME} - # response = client.data_connection_service_list_data_connection_artifacts( - # project_id=project_id, - # id=response.id, - # ) - except lightning_cloud.openapi.rest.ApiException as e: - message = ast.literal_eval(e.body.decode("utf-8"))["message"] - _error_and_exit(f"The data connection creation failed. Message: {message}") - - rich.print(f"[green]Succeeded[/green]: You have created a new data connection {name}.") - return None diff --git a/src/lightning/app/cli/core.py b/src/lightning/app/cli/core.py deleted file mode 100644 index 6d54c31426ee1..0000000000000 --- a/src/lightning/app/cli/core.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import abc - -from rich.table import Table - - -class Formatable(abc.ABC): - @abc.abstractmethod - def as_table(self) -> Table: - pass - - @abc.abstractmethod - def as_json(self) -> str: - pass diff --git a/src/lightning/app/cli/lightning_cli.py b/src/lightning/app/cli/lightning_cli.py deleted file mode 100644 index 6aa84063ab93f..0000000000000 --- a/src/lightning/app/cli/lightning_cli.py +++ /dev/null @@ -1,503 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -from pathlib import Path -from typing import Tuple, Union - -import click -from requests.exceptions import ConnectionError - -import lightning.app.core.constants as constants -from lightning.app import __version__ as ver -from lightning.app.cli import cmd_init, cmd_install, cmd_pl_init, cmd_react_ui_init -from lightning.app.cli.commands.app_commands import _run_app_command -from lightning.app.cli.commands.cd import cd -from lightning.app.cli.commands.cp import cp -from lightning.app.cli.commands.logs import logs -from lightning.app.cli.commands.ls import ls -from lightning.app.cli.commands.pwd import pwd -from lightning.app.cli.commands.rm import rm -from lightning.app.cli.connect.app import ( - _list_app_commands, - _retrieve_connection_to_an_app, - connect_app, - disconnect_app, -) -from lightning.app.cli.connect.data import connect_data -from lightning.app.cli.lightning_cli_delete import delete -from lightning.app.cli.lightning_cli_launch import launch -from lightning.app.cli.lightning_cli_list import get_list -from lightning.app.core.constants import ( - APP_SERVER_HOST, - APP_SERVER_PORT, - ENABLE_APP_COMMENT_COMMAND_EXECUTION, - get_lightning_cloud_url, -) -from lightning.app.launcher.launcher import ( - run_lightning_flow, - run_lightning_work, - serve_frontend, - start_application_server, - start_flow_and_servers, -) -from lightning.app.runners.cloud import CloudRuntime -from lightning.app.runners.runtime import dispatch -from lightning.app.runners.runtime_type import RuntimeType -from lightning.app.utilities.app_commands import run_app_commands -from lightning.app.utilities.app_helpers import Logger -from lightning.app.utilities.cli_helpers import ( - _check_environment_and_redirect, - _check_version_and_upgrade, - _format_input_env_variables, -) -from lightning.app.utilities.exceptions import _ApiExceptionHandler -from lightning.app.utilities.login import Auth -from lightning.app.utilities.port import _find_lit_app_port - -logger = Logger(__name__) - - -def main() -> None: - # Check environment and versions if not in the cloud and not testing - is_testing = bool(int(os.getenv("LIGHTING_TESTING", "0"))) - if not is_testing and "LIGHTNING_APP_STATE_URL" not in os.environ: - try: - # Enforce running in PATH Python - _check_environment_and_redirect() - - # Check for newer versions and upgrade - _check_version_and_upgrade() - except SystemExit: - raise - except Exception: - # Note: We intentionally ignore all exceptions here so that we never panic if one of the above calls fails. - # If they fail for some reason users should still be able to continue with their command. - click.echo( - "We encountered an unexpected problem while checking your environment." - "We will still proceed with the command, however, there is a chance that errors may occur." - ) - - # 1: Handle connection to a Lightning App. - if len(sys.argv) > 1 and sys.argv[1] in ("connect", "disconnect", "logout"): - _main() - else: - # 2: Collect the connection a Lightning App. - app_name, app_id = _retrieve_connection_to_an_app() - if app_name: - # 3: Handle development use case. - is_local_app = app_name == "localhost" - if sys.argv[1:3] == ["run", "app"] or ( - sys.argv[1:3] == ["show", "logs"] and "show logs" not in _list_app_commands(False) - ): - _main() - else: - if is_local_app: - message = "You are connected to the local Lightning App." - else: - message = f"You are connected to the cloud Lightning App: {app_name}." - - if (len(sys.argv) > 1 and sys.argv[1] in ["-h", "--help"]) or len(sys.argv) == 1: - _list_app_commands() - else: - _run_app_command(app_name, app_id) - - click.echo() - click.echo(message + " Return to the primary CLI with `lightning_app disconnect`.") - else: - _main() - - -@click.group(cls=_ApiExceptionHandler) -@click.version_option(ver) -def _main() -> None: - pass - - -@_main.group() -def show() -> None: - """Show given resource.""" - pass - - -@_main.group() -def connect() -> None: - """Connect apps and data.""" - pass - - -@_main.group() -def disconnect() -> None: - """Disconnect apps.""" - pass - - -connect.command("app")(connect_app) -disconnect.command("app")(disconnect_app) -connect.command("data", hidden=True)(connect_data) -_main.command(hidden=True)(ls) -_main.command(hidden=True)(cd) -_main.command(hidden=True)(cp) -_main.command(hidden=True)(pwd) -_main.command(hidden=True)(rm) -show.command()(logs) - - -@_main.command() -def login() -> None: - """Log in to your lightning.ai account.""" - auth = Auth() - auth.clear() - - try: - auth.authenticate() - except ConnectionError: - click.echo(f"Unable to connect to {get_lightning_cloud_url()}. Please check your internet connection.") - exit(1) - - -@_main.command() -def logout() -> None: - """Log out of your lightning.ai account.""" - Auth().clear() - disconnect_app(logout=True) - - -def _run_app( - file: str, - cloud: bool, - without_server: bool, - no_cache: bool, - name: str, - blocking: bool, - open_ui: bool, - env: tuple, - secret: tuple, - run_app_comment_commands: bool, - enable_basic_auth: str, -) -> None: - if not os.path.exists(file): - original_file = file - file = cmd_install.gallery_apps_and_components(file, True, "latest", overwrite=True) # type: ignore[assignment] # E501 - if file is None: - click.echo(f"The provided entrypoint `{original_file}` doesn't exist.") - sys.exit(1) - run_app_comment_commands = True - - runtime_type = RuntimeType.CLOUD if cloud else RuntimeType.MULTIPROCESS - - # Cloud specific validations - if runtime_type != RuntimeType.CLOUD: - if no_cache: - raise click.ClickException( - "Caching is a property of apps running in cloud. " - "Using the flag --no-cache in local execution is not supported." - ) - if secret: - raise click.ClickException( - "Secrets can only be used for apps running in cloud. " - "Using the option --secret in local execution is not supported." - ) - if (ENABLE_APP_COMMENT_COMMAND_EXECUTION or run_app_comment_commands) and file is not None: - run_app_commands(str(file)) - - env_vars = _format_input_env_variables(env) - os.environ.update(env_vars) - - secrets = _format_input_env_variables(secret) - - port = _find_lit_app_port(constants.APP_SERVER_PORT) - constants.APP_SERVER_PORT = port - - click.echo("Your Lightning App is starting. This won't take long.") - - # TODO: Fixme when Grid utilities are available. - # And refactor test_lightning_run_app_cloud - file_path = Path(file) - dispatch( - file_path, - runtime_type, - start_server=not without_server, - no_cache=no_cache, - blocking=blocking, - open_ui=open_ui, - name=name, - env_vars=env_vars, - secrets=secrets, - run_app_comment_commands=run_app_comment_commands, - enable_basic_auth=enable_basic_auth, - port=port, - ) - if runtime_type == RuntimeType.CLOUD: - click.echo("Application is ready in the cloud") - - -@_main.group() -def run() -> None: - """Run a Lightning application locally or on the cloud.""" - - -@run.command("app") -@click.argument("file", type=str) -@click.option("--cloud", type=bool, default=False, is_flag=True) -@click.option("--name", help="The current application name", default="", type=str) -@click.option("--without-server", is_flag=True, default=False) -@click.option( - "--no-cache", - is_flag=True, - default=False, - help="Disable caching of packages " "installed from requirements.txt", -) -@click.option("--blocking", "blocking", type=bool, default=False) -@click.option( - "--open-ui", - type=bool, - default=True, - help="Decide whether to launch the app UI in a web browser", -) -@click.option("--env", type=str, default=[], multiple=True, help="Environment variables to be set for the app.") -@click.option("--secret", type=str, default=[], multiple=True, help="Secret variables to be set for the app.") -@click.option("--app_args", type=str, default=[], multiple=True, help="Collection of arguments for the app.") -@click.option( - "--setup", - "-s", - "run_app_comment_commands", - is_flag=True, - default=False, - help="run environment setup commands from the app comments.", -) -@click.option( - "--enable-basic-auth", - type=str, - default="", - help="Enable basic authentication for the app and use credentials provided in the format username:password", -) -def run_app( - file: str, - cloud: bool, - without_server: bool, - no_cache: bool, - name: str, - blocking: bool, - open_ui: bool, - env: tuple, - secret: tuple, - app_args: tuple, - run_app_comment_commands: bool, - enable_basic_auth: str, -) -> None: - """Run an app from a file.""" - _run_app( - file, - cloud, - without_server, - no_cache, - name, - blocking, - open_ui, - env, - secret, - run_app_comment_commands, - enable_basic_auth, - ) - - -@_main.command("open", hidden=True) -@click.argument("path", type=str, default=".") -@click.option("--name", help="The name to use for the CloudSpace", default="", type=str) -def open(path: str, name: str) -> None: - """Open files or folders from your machine on the cloud.""" - if not os.path.exists(path): - click.echo(f"The provided path `{path}` doesn't exist.") - sys.exit(1) - - runtime = CloudRuntime(entrypoint=Path(path)) - runtime.open(name) - - -_main.add_command(get_list) -_main.add_command(delete) -_main.add_command(launch) -_main.add_command(cmd_install.install) - - -@_main.group() -def init() -> None: - """Init a Lightning App and/or component.""" - - -@init.command("app") -@click.argument("name", type=str, required=False) -def init_app(name: str) -> None: - cmd_init.app(name) - - -@init.command("pl-app") -@click.argument("source", nargs=-1) -@click.option( - "--name", - "-n", - type=str, - default="pl-app", - help="The name of the folder where the app code will be. Default: pl-app", -) -@click.option( - "--overwrite", - "-f", - is_flag=True, - default=False, - help="When set, overwrite the output directory without asking if it already exists.", -) -def init_pl_app(source: Union[Tuple[str], Tuple[str, str]], name: str, overwrite: bool = False) -> None: - """Create an app from your PyTorch Lightning source files.""" - if len(source) == 1: - script_path = source[0] - source_dir = str(Path(script_path).resolve().parent) - elif len(source) == 2: - # enable type checking once https://github.com/python/mypy/issues/1178 is available - source_dir, script_path = source - else: - click.echo( - f"Incorrect number of arguments. You passed ({', '.join(source)}) but only either one argument" - f" (script path) or two arguments (root dir, script path) are allowed. Examples:\n" - f"lightning init pl-app ./path/to/script.py\n" - f"lightning init pl-app ./code ./code/path/to/script.py", - err=True, - ) - raise SystemExit(1) - - cmd_pl_init.pl_app(source_dir=source_dir, script_path=script_path, name=name, overwrite=overwrite) - - -@init.command("component") -@click.argument("name", type=str, required=False) -def init_component(name: str) -> None: - cmd_init.component(name) - - -@init.command("react-ui") -@click.option( - "--dest_dir", - "-dest_dir", - type=str, - help="optional destination directory to create the react ui", -) -def init_react_ui(dest_dir: str) -> None: - """Create a react UI to give a Lightning component a React.js web user interface (UI)""" - cmd_react_ui_init.react_ui(dest_dir) - - -def _prepare_file(file: str) -> str: - exists = os.path.exists(file) - if exists: - return file - - raise FileNotFoundError(f"The provided file {file} hasn't been found.") - - -@run.command("server") -@click.argument("file", type=click.Path(exists=True)) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -@click.option("--host", help="Application running host", default=APP_SERVER_HOST, type=str) -@click.option("--port", help="Application running port", default=APP_SERVER_PORT, type=int) -def run_server(file: str, queue_id: str, host: str, port: int) -> None: - """It takes the application file as input, build the application object and then use that to run the application - server. - - This is used by the cloud runners to start the status server for the application - - """ - logger.debug(f"Run Server: {file} {queue_id} {host} {port}") - start_application_server(file, host, port, queue_id=queue_id) - - -@run.command("flow") -@click.argument("file", type=click.Path(exists=True)) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -@click.option("--base-url", help="Base url at which the app server is hosted", default="") -def run_flow(file: str, queue_id: str, base_url: str) -> None: - """It takes the application file as input, build the application object, proxy all the work components and then run - the application flow defined in the root component. - - It does exactly what a singleprocess dispatcher would do but with proxied work components. - - """ - logger.debug(f"Run Flow: {file} {queue_id} {base_url}") - run_lightning_flow(file, queue_id=queue_id, base_url=base_url) - - -@run.command("work") -@click.argument("file", type=click.Path(exists=True)) -@click.option("--work-name", type=str) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -def run_work(file: str, work_name: str, queue_id: str) -> None: - """Unlike other entrypoints, this command will take the file path or module details for a work component and run - that by fetching the states from the queues.""" - logger.debug(f"Run Work: {file} {work_name} {queue_id}") - run_lightning_work( - file=file, - work_name=work_name, - queue_id=queue_id, - ) - - -@run.command("frontend") -@click.argument("file", type=click.Path(exists=True)) -@click.option("--flow-name") -@click.option("--host") -@click.option("--port", type=int) -def run_frontend(file: str, flow_name: str, host: str, port: int) -> None: - """Serve the frontend specified by the given flow.""" - logger.debug(f"Run Frontend: {file} {flow_name} {host}") - serve_frontend(file=file, flow_name=flow_name, host=host, port=port) - - -@run.command("flow-and-servers") -@click.argument("file", type=click.Path(exists=True)) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -@click.option("--base-url", help="Base url at which the app server is hosted", default="") -@click.option("--host", help="Application running host", default=APP_SERVER_HOST, type=str) -@click.option("--port", help="Application running port", default=APP_SERVER_PORT, type=int) -@click.option( - "--flow-port", - help="Pair of flow name and frontend port", - type=(str, int), - multiple=True, -) -def run_flow_and_servers( - file: str, - base_url: str, - queue_id: str, - host: str, - port: int, - flow_port: Tuple[Tuple[str, int]], -) -> None: - """It takes the application file as input, build the application object and then use that to run the application - flow defined in the root component, the application server and all the flow frontends. - - This is used by the cloud runners to start the flow, the status server and all frontends for the application - - """ - logger.debug(f"Run Flow: {file} {queue_id} {base_url}") - logger.debug(f"Run Server: {file} {queue_id} {host} {port}.") - logger.debug(f"Run Frontend's: {flow_port}") - start_flow_and_servers( - entrypoint_file=file, - base_url=base_url, - queue_id=queue_id, - host=host, - port=port, - flow_names_and_ports=flow_port, - ) diff --git a/src/lightning/app/cli/lightning_cli_delete.py b/src/lightning/app/cli/lightning_cli_delete.py deleted file mode 100644 index 179e5b6fc365d..0000000000000 --- a/src/lightning/app/cli/lightning_cli_delete.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import click -import inquirer -from inquirer.themes import GreenPassion -from rich.console import Console - -from lightning.app.cli.cmd_apps import _AppManager - - -@click.group("delete") -def delete() -> None: - """Delete Lightning AI self-managed resources (e.g. apps)""" - pass - - -def _find_selected_app_instance_id(app_name: str) -> str: - console = Console() - app_manager = _AppManager() - - all_app_names_and_ids = {} - selected_app_instance_id = None - - for app in app_manager.list_apps(): - all_app_names_and_ids[app.name] = app.id - # figure out the ID of some app_name - if app_name == app.name or app_name == app.id: - selected_app_instance_id = app.id - break - - if selected_app_instance_id is None: - # when there is no app with the given app_name, - # ask the user which app they would like to delete. - console.print(f'[b][yellow]Cannot find app named "{app_name}"[/yellow][/b]') - try: - ask = [ - inquirer.List( - "app_name", - message="Select the app name to delete", - choices=list(all_app_names_and_ids.keys()), - ), - ] - app_name = inquirer.prompt(ask, theme=GreenPassion(), raise_keyboard_interrupt=True)["app_name"] - selected_app_instance_id = all_app_names_and_ids[app_name] - except KeyboardInterrupt: - console.print("[b][red]Cancelled by user![/b][/red]") - raise InterruptedError - - return selected_app_instance_id - - -def _delete_app_confirmation_prompt(app_name: str) -> None: - console = Console() - - # when the --yes / -y flags were not passed, do a final - # confirmation that the user wants to delete the app. - try: - ask = [ - inquirer.Confirm( - "confirm", - message=f'Are you sure you want to delete app "{app_name}""?', - default=False, - ), - ] - if inquirer.prompt(ask, theme=GreenPassion(), raise_keyboard_interrupt=True)["confirm"] is False: - console.print("[b][red]Aborted![/b][/red]") - raise InterruptedError - except KeyboardInterrupt: - console.print("[b][red]Cancelled by user![/b][/red]") - raise InterruptedError - - -@delete.command("app") -@click.argument("app-name", type=str) -@click.option( - "skip_user_confirm_prompt", - "--yes", - "-y", - is_flag=True, - default=False, - help="Do not prompt for confirmation.", -) -def delete_app(app_name: str, skip_user_confirm_prompt: bool) -> None: - """Delete a Lightning app. - - Deleting an app also deletes all app websites, works, artifacts, and logs. This permanently removes any record of - the app as well as all any of its associated resources and data. This does not affect any resources and data - associated with other Lightning apps on your account. - - """ - console = Console() - - try: - selected_app_instance_id = _find_selected_app_instance_id(app_name=app_name) - if not skip_user_confirm_prompt: - _delete_app_confirmation_prompt(app_name=app_name) - except InterruptedError: - return - - try: - # Delete the app! - app_manager = _AppManager() - app_manager.delete(app_id=selected_app_instance_id) - except Exception as ex: - console.print( - f'[b][red]An issue occurred while deleting app "{app_name}. If the issue persists, please ' - "reach out to us at [link=mailto:support@lightning.ai]support@lightning.ai[/link][/b][/red]." - ) - raise click.ClickException(str(ex)) - - console.print(f'[b][green]App "{app_name}" has been successfully deleted"![/green][/b]') - return diff --git a/src/lightning/app/cli/lightning_cli_launch.py b/src/lightning/app/cli/lightning_cli_launch.py deleted file mode 100644 index c171fd7b946f1..0000000000000 --- a/src/lightning/app/cli/lightning_cli_launch.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import Tuple - -import click - -from lightning.app.core.constants import APP_SERVER_HOST, APP_SERVER_PORT -from lightning.app.launcher.launcher import ( - run_lightning_flow, - run_lightning_work, - serve_frontend, - start_application_server, - start_flow_and_servers, -) - -logger = logging.getLogger(__name__) - - -@click.group(name="launch", hidden=True) -def launch() -> None: - """Launch your application.""" - - -@launch.command("server", hidden=True) -@click.argument("file", type=click.Path(exists=True)) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -@click.option("--host", help="Application running host", default=APP_SERVER_HOST, type=str) -@click.option("--port", help="Application running port", default=APP_SERVER_PORT, type=int) -def run_server(file: str, queue_id: str, host: str, port: int) -> None: - """It takes the application file as input, build the application object and then use that to run the application - server. - - This is used by the cloud runners to start the status server for the application - - """ - logger.debug(f"Run Server: {file} {queue_id} {host} {port}") - start_application_server(file, host, port, queue_id=queue_id) - - -@launch.command("flow", hidden=True) -@click.argument("file", type=click.Path(exists=True)) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -@click.option("--base-url", help="Base url at which the app server is hosted", default="") -def run_flow(file: str, queue_id: str, base_url: str) -> None: - """It takes the application file as input, build the application object, proxy all the work components and then run - the application flow defined in the root component. - - It does exactly what a singleprocess dispatcher would do but with proxied work components. - - """ - logger.debug(f"Run Flow: {file} {queue_id} {base_url}") - run_lightning_flow(file, queue_id=queue_id, base_url=base_url) - - -@launch.command("work", hidden=True) -@click.argument("file", type=click.Path(exists=True)) -@click.option("--work-name", type=str) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -def run_work(file: str, work_name: str, queue_id: str) -> None: - """Unlike other entrypoints, this command will take the file path or module details for a work component and run - that by fetching the states from the queues.""" - logger.debug(f"Run Work: {file} {work_name} {queue_id}") - run_lightning_work( - file=file, - work_name=work_name, - queue_id=queue_id, - ) - - -@launch.command("frontend", hidden=True) -@click.argument("file", type=click.Path(exists=True)) -@click.option("--flow-name") -@click.option("--host") -@click.option("--port", type=int) -def run_frontend(file: str, flow_name: str, host: str, port: int) -> None: - """Serve the frontend specified by the given flow.""" - logger.debug(f"Run Frontend: {file} {flow_name} {host}") - serve_frontend(file=file, flow_name=flow_name, host=host, port=port) - - -@launch.command("flow-and-servers", hidden=True) -@click.argument("file", type=click.Path(exists=True)) -@click.option("--queue-id", help="ID for identifying queue", default="", type=str) -@click.option("--base-url", help="Base url at which the app server is hosted", default="") -@click.option("--host", help="Application running host", default=APP_SERVER_HOST, type=str) -@click.option("--port", help="Application running port", default=APP_SERVER_PORT, type=int) -@click.option( - "--flow-port", - help="Pair of flow name and frontend port", - type=(str, int), - multiple=True, -) -def run_flow_and_servers( - file: str, - base_url: str, - queue_id: str, - host: str, - port: int, - flow_port: Tuple[Tuple[str, int]], -) -> None: - """It takes the application file as input, build the application object and then use that to run the application - flow defined in the root component, the application server and all the flow frontends. - - This is used by the cloud runners to start the flow, the status server and all frontends for the application - - """ - logger.debug(f"Run Flow: {file} {queue_id} {base_url}") - logger.debug(f"Run Server: {file} {queue_id} {host} {port}.") - logger.debug(f"Run Frontend's: {flow_port}") - start_flow_and_servers( - entrypoint_file=file, - base_url=base_url, - queue_id=queue_id, - host=host, - port=port, - flow_names_and_ports=flow_port, - ) diff --git a/src/lightning/app/cli/lightning_cli_list.py b/src/lightning/app/cli/lightning_cli_list.py deleted file mode 100644 index 0cbc8e3cc1887..0000000000000 --- a/src/lightning/app/cli/lightning_cli_list.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright The Lightning AI team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any - -import click - -from lightning.app.cli.cmd_apps import _AppManager - - -@click.group(name="list") -def get_list() -> None: - """List Lightning AI self-managed resources (e.g. apps)""" - pass - - -@get_list.command("apps") -def list_apps(**kwargs: Any) -> None: - """List your Lightning AI apps.""" - app_manager = _AppManager() - app_manager.list() diff --git a/src/lightning/app/cli/pl-app-template/.gitignore b/src/lightning/app/cli/pl-app-template/.gitignore deleted file mode 100644 index 01aa0091c3945..0000000000000 --- a/src/lightning/app/cli/pl-app-template/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.storage diff --git a/src/lightning/app/cli/pl-app-template/.lightningignore b/src/lightning/app/cli/pl-app-template/.lightningignore deleted file mode 100644 index 5895fd5187660..0000000000000 --- a/src/lightning/app/cli/pl-app-template/.lightningignore +++ /dev/null @@ -1,2 +0,0 @@ -.storage -ui/node_modules diff --git a/src/lightning/app/cli/pl-app-template/app.py b/src/lightning/app/cli/pl-app-template/app.py deleted file mode 100644 index b15ea21a02276..0000000000000 --- a/src/lightning/app/cli/pl-app-template/app.py +++ /dev/null @@ -1,105 +0,0 @@ -import os -from typing import Dict, List, Optional, Union - -from core.components import TensorBoard, WeightsAndBiases -from core.components.script_runner import ScriptRunner -from lightning.app import LightningApp, LightningFlow -from lightning.app.frontend import StaticWebFrontend -from lightning.app.storage.path import Path -from lightning.app.utilities.packaging.cloud_compute import CloudCompute - - -class ReactUI(LightningFlow): - def configure_layout(self): - return StaticWebFrontend(str(Path(__file__).parent / "ui/build")) - - -class ScriptOrchestrator(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.script_runner: Optional[ScriptRunner] = None - self.triggered: bool = False - self.running: bool = False - self.succeeded: bool = False - self.failed: bool = False - self.script_args: List[str] = [] - self.cloud_compute_args: Dict[str, Union[str, int]] = {"name": "cpu-small"} - self.environment_variables: Dict[str, str] = {} - self.script_path = "{{ script_path }}" - - def run(self) -> None: - if not self.triggered: - return - - if self.script_runner is None: - self.script_runner = ScriptRunner( - root_path=str(Path(__file__).parent / "source"), - script_path=str(Path(__file__).parent / "source" / self.script_path), - script_args=self.script_args, - env=self._prepare_environment(), - parallel=True, - cloud_compute=CloudCompute(**self.cloud_compute_args), - raise_exception=False, - ) - self.script_runner.run() - - self.running = self.script_runner is not None and self.script_runner.has_started - self.succeeded = self.script_runner is not None and self.script_runner.has_succeeded - self.failed = self.script_runner is not None and self.script_runner.has_failed - - if self.succeeded or self.failed: - self.triggered = False - # TODO: support restarting - # self.script_runner = None - - def _prepare_environment(self) -> Dict[str, str]: - env = os.environ.copy() - env.update(self.environment_variables) - return env - - -class Main(LightningFlow): - def __init__(self) -> None: - super().__init__() - self.react_ui = ReactUI() - self.script_orchestrator = ScriptOrchestrator() - self.running_in_cloud = bool(os.environ.get("LIGHTNING_CLOUD_APP_ID", False)) - - def run(self) -> None: - self.react_ui.run() - self.script_orchestrator.run() - - if self.script_orchestrator.script_runner and self.script_orchestrator.script_runner.logger_metadatas: - if not getattr(self, "logger_component", None): - # TODO: Hack with hasattr and setattr until - # https://linear.app/gridai/issue/LAI2-8970/work-getting-set-to-none-in-state-update-from-appstate - # is resolved - logger_component = self._choose_logger_component() - if logger_component is not None: - setattr(self, "logger_component", logger_component) - else: - self.logger_component.run() - - def configure_layout(self): - tabs = [{"name": "Home", "content": self.react_ui}] - if hasattr(self, "logger_component"): - tabs.extend(self.logger_component.configure_layout()) - return tabs - - def _choose_logger_component(self) -> Optional[Union[TensorBoard, WeightsAndBiases]]: - logger_metadatas = self.script_orchestrator.script_runner.logger_metadatas - if not logger_metadatas: - return None - if logger_metadatas[0].get("class_name") == "TensorBoardLogger": - return TensorBoard(log_dir=self.script_orchestrator.script_runner.log_dir) - if logger_metadatas[0].get("class_name") == "WandbLogger": - return WeightsAndBiases( - username=logger_metadatas[0]["username"], - project_name=logger_metadatas[0]["project_name"], - run_id=logger_metadatas[0]["run_id"], - api_key=self.script_orchestrator.environment_variables.get("WANDB_API_KEY"), - ) - return None - - -app = LightningApp(Main()) diff --git a/src/lightning/app/cli/pl-app-template/core/__init__.py b/src/lightning/app/cli/pl-app-template/core/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/pl-app-template/core/callbacks.py b/src/lightning/app/cli/pl-app-template/core/callbacks.py deleted file mode 100644 index 87ec8e9bcbda2..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/callbacks.py +++ /dev/null @@ -1,319 +0,0 @@ -import inspect -from typing import TYPE_CHECKING, Any, Dict, Union - -import lightning.pytorch as pl -from lightning.app.storage.path import Path -from lightning.app.utilities.app_helpers import Logger -from lightning.pytorch import Callback -from lightning.pytorch.callbacks.progress.progress_bar import get_standard_metrics -from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger -from lightning.pytorch.utilities.parsing import collect_init_args - -from core.state import ProgressBarState, TrainerState - -if TYPE_CHECKING: - from core.components.script_runner import ScriptRunner - - -_log = Logger(__name__) - - -class PLAppProgressTracker(Callback): - """This callback tracks and communicates the Trainer's progress to the running PyTorch Lightning App.""" - - def __init__(self, work: "ScriptRunner", refresh_rate: int = 1) -> None: - super().__init__() - self.work = work - self.refresh_rate = refresh_rate - self.is_enabled = False - self._state = ProgressBarState() - - def setup( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - stage: str, - ) -> None: - self.is_enabled = trainer.is_global_zero - - def on_train_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - # We calculate the estimated stepping batches here instead of in the setup hook, because calling the - # `Trainer.estimated_stepping_batches` too early would lead to a barrier() call in case of DDP and since this - # callback is only attached on rank 0, would lead to a stall. - self._state.fit.estimated_stepping_batches = trainer.estimated_stepping_batches - - def on_train_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", *_: Any) -> None: - self._state.fit.total_train_batches = self._total_train_batches(trainer) - self._state.fit.total_val_batches = self._total_val_batches(trainer) - self._state.fit.current_epoch = trainer.current_epoch - if self.is_enabled: - self._send_state() - - def on_train_batch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", *_: Any) -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - current = self._train_batch_idx(trainer) - self._state.fit.train_batch_idx = current - self._state.fit.global_step = trainer.global_step - if self._should_send(current, self._total_train_batches(trainer)): - self._send_state() - - def on_train_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - if self.is_enabled: - self._send_state() - - def on_validation_batch_start( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - batch: Any, - batch_idx: int, - dataloader_idx: int, - ) -> None: - if trainer.state.fn == "fit": - self._state.fit.val_dataloader_idx = dataloader_idx - self._state.fit.total_val_batches = self._total_val_batches(trainer) - if trainer.state.fn == "validate": - self._state.val.dataloader_idx = dataloader_idx - self._state.val.total_val_batches = self._total_val_batches(trainer) - - def on_validation_batch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", *_: Any) -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - current = self._val_batch_idx(trainer) - if trainer.state.fn == "fit": - self._state.fit.val_batch_idx = current - if trainer.state.fn == "validate": - self._state.val.val_batch_idx = current - if self._should_send(current, self._total_val_batches(trainer)): - self._send_state() - - def on_validation_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - if self.is_enabled: - self._send_state() - - def on_test_batch_start( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - batch: Any, - batch_idx: int, - dataloader_idx: int, - ) -> None: - self._state.test.dataloader_idx = dataloader_idx - self._state.test.total_test_batches = trainer.num_test_batches[dataloader_idx] - - def on_test_batch_end( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - outputs: Any, - batch: Any, - batch_idx: int, - dataloader_idx: int = 0, - ) -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - current = self._test_batch_idx(trainer) - self._state.test.test_batch_idx = current - if self._should_send(current, trainer.num_test_batches[dataloader_idx]): - self._send_state() - - def on_test_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - if self.is_enabled: - self._send_state() - - def on_predict_batch_start( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - batch: Any, - batch_idx: int, - dataloader_idx: int, - ) -> None: - self._state.predict.dataloader_idx = dataloader_idx - self._state.predict.total_predict_batches = trainer.num_predict_batches[dataloader_idx] - - def on_predict_batch_end( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - outputs: Any, - batch: Any, - batch_idx: int, - dataloader_idx: int = 0, - ) -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - current = self._predict_batch_idx(trainer) - self._state.predict.predict_batch_idx = current - if self._should_send(current, trainer.num_predict_batches[dataloader_idx]): - self._send_state() - - def on_predict_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.metrics = self._progress_bar_metrics(trainer, pl_module) - if self.is_enabled: - self._send_state() - - def _train_batch_idx(self, trainer: "pl.Trainer") -> int: - return trainer.fit_loop.epoch_loop.batch_progress.current.processed - - def _val_batch_idx(self, trainer: "pl.Trainer") -> int: - loop = trainer.fit_loop.epoch_loop.val_loop if trainer.state.fn == "fit" else trainer.validate_loop - - return loop.epoch_loop.batch_progress.current.processed - - def _test_batch_idx(self, trainer: "pl.Trainer") -> int: - return trainer.test_loop.epoch_loop.batch_progress.current.processed - - def _predict_batch_idx(self, trainer: "pl.Trainer") -> int: - return trainer.predict_loop.epoch_loop.batch_progress.current.processed - - def _total_train_batches(self, trainer: "pl.Trainer") -> Union[int, float]: - return trainer.num_training_batches - - def _total_val_batches(self, trainer: "pl.Trainer") -> Union[int, float]: - return sum(trainer.num_val_batches) if trainer.fit_loop.epoch_loop._should_check_val_epoch() else 0 - - def _progress_bar_metrics( - self, trainer: "pl.Trainer", pl_module: "pl.LightningModule" - ) -> Dict[str, Union[str, float]]: - standard_metrics = get_standard_metrics(trainer, pl_module) - pbar_metrics = trainer.progress_bar_metrics - return {**standard_metrics, **pbar_metrics} - - def _send_state(self) -> None: - self.work.trainer_progress = self._state.dict() - - def _should_send(self, current: int, total: int) -> bool: - return self.is_enabled and current % self.refresh_rate == 0 or current == total - - -class PLAppTrainerStateTracker(Callback): - def __init__(self, work: "ScriptRunner") -> None: - super().__init__() - self.work = work - self._state = TrainerState() - - def on_fit_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.fn = "fit" - self.work.trainer_state = self._state.dict() - - def on_fit_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.fn = None - self.work.trainer_state = self._state.dict() - - def on_train_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.stage = "training" - self.work.trainer_state = self._state.dict() - - def on_train_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.stage = None - self.work.trainer_state = self._state.dict() - - def on_validation_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.stage = "validating" - self.work.trainer_state = self._state.dict() - - def on_validation_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.stage = None - self.work.trainer_state = self._state.dict() - - def on_test_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.fn = "test" - self._state.stage = "testing" - self.work.trainer_state = self._state.dict() - - def on_test_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.fn = None - self._state.stage = None - self.work.trainer_state = self._state.dict() - - def on_predict_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.fn = "predict" - self._state.stage = "predicting" - self.work.trainer_state = self._state.dict() - - def on_predict_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - self._state.fn = None - self._state.stage = None - self.work.trainer_state = self._state.dict() - - -class PLAppSummary(Callback): - def __init__(self, work: "ScriptRunner") -> None: - super().__init__() - self.work = work - - def on_init_end(self, trainer: "pl.Trainer") -> None: - current_frame = inspect.currentframe() - # Trainer.init() -> Trainer._call_callback_hooks() -> Callback.on_init_end() - frame = current_frame.f_back.f_back - init_args = {} - for local_args in collect_init_args(frame, []): - init_args.update(local_args) - - self.work.trainer_hparams = self._sanitize_trainer_init_args(init_args) - - def setup( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - stage: str, - ) -> None: - self.work.model_hparams = self._sanitize_model_init_args(dict(**pl_module.hparams)) - - def _sanitize_trainer_init_args(self, init_args: Dict[str, Any]) -> Dict[str, str]: - if init_args["callbacks"]: - init_args["callbacks"] = [c.__class__.__name__ for c in init_args["callbacks"]] - return {k: str(v) for k, v in init_args.items()} - - def _sanitize_model_init_args(self, init_args: Dict[str, Any]) -> Dict[str, str]: - return {k: str(v) for k, v in init_args.items()} - - -class PLAppArtifactsTracker(Callback): - def __init__(self, work: "ScriptRunner") -> None: - super().__init__() - self.work = work - - def setup( - self, - trainer: "pl.Trainer", - pl_module: "pl.LightningModule", - stage: str, - ) -> None: - log_dir = self._get_logdir(trainer) - self.work.log_dir = Path(log_dir) if log_dir is not None else None - self._collect_logger_metadata(trainer) - - def on_train_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None: - if trainer.checkpoint_callback and trainer.checkpoint_callback.dirpath is not None: - self.work.checkpoint_dir = Path(trainer.checkpoint_callback.dirpath) - - def _collect_logger_metadata(self, trainer: "pl.Trainer") -> None: - if not trainer.loggers: - return - - for logger in trainer.loggers: - metadata = {"class_name": logger.__class__.__name__} - if isinstance(logger, WandbLogger) and not logger._offline: - metadata.update({ - "username": logger.experiment.entity, - "project_name": logger.name, - "run_id": logger.version, - }) - - if metadata and metadata not in self.work.logger_metadatas: - self.work.logger_metadatas.append(metadata) - - @staticmethod - def _get_logdir(trainer: "pl.Trainer") -> str: - """The code here is the same as in the ``Trainer.log_dir``, with the exception of the broadcast call.""" - if len(trainer.loggers) == 1: - if isinstance(trainer.logger, TensorBoardLogger): - dirpath = trainer.logger.log_dir - else: - dirpath = trainer.logger.save_dir - else: - dirpath = trainer.default_root_dir - return dirpath diff --git a/src/lightning/app/cli/pl-app-template/core/components/__init__.py b/src/lightning/app/cli/pl-app-template/core/components/__init__.py deleted file mode 100644 index 75f49eb7da05e..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/components/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from core.components.logger.tensorboard import TensorBoard # noqa: F401 -from core.components.logger.weights_and_biases import WeightsAndBiases # noqa: F401 diff --git a/src/lightning/app/cli/pl-app-template/core/components/logger/__init__.py b/src/lightning/app/cli/pl-app-template/core/components/logger/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/pl-app-template/core/components/logger/tensorboard.py b/src/lightning/app/cli/pl-app-template/core/components/logger/tensorboard.py deleted file mode 100644 index 0e1a536ff4859..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/components/logger/tensorboard.py +++ /dev/null @@ -1,49 +0,0 @@ -import subprocess -import time -from typing import Dict, List - -from lightning.app import BuildConfig, LightningFlow, LightningWork -from lightning.app.storage.path import Path - - -class TensorBoard(LightningFlow): - def __init__(self, log_dir: Path, sync_every_n_seconds: int = 5) -> None: - """This TensorBoard component synchronizes the log directory of an experiment and starts up the server. - - Args: - log_dir: The path to the directory where the TensorBoard log-files will appear. - sync_every_n_seconds: How often to sync the log directory (given as an argument to the run method) - - """ - super().__init__() - self.worker = TensorBoardWorker(log_dir=log_dir, sync_every_n_seconds=sync_every_n_seconds) - - def run(self) -> None: - self.worker.run() - - def configure_layout(self) -> List[Dict[str, str]]: - return [{"name": "Training Logs", "content": self.worker.url}] - - -class TensorBoardWorker(LightningWork): - def __init__(self, log_dir: Path, sync_every_n_seconds: int = 5) -> None: - super().__init__(cloud_build_config=BuildConfig(requirements=["tensorboard"])) - self.log_dir = log_dir - self._sync_every_n_seconds = sync_every_n_seconds - - def run(self) -> None: - subprocess.Popen([ - "tensorboard", - "--logdir", - str(self.log_dir), - "--host", - self.host, - "--port", - str(self.port), - ]) - - # Download the log directory periodically - while True: - time.sleep(self._sync_every_n_seconds) - if self.log_dir.exists_remote(): - self.log_dir.get(overwrite=True) diff --git a/src/lightning/app/cli/pl-app-template/core/components/logger/weights_and_biases.py b/src/lightning/app/cli/pl-app-template/core/components/logger/weights_and_biases.py deleted file mode 100644 index bf20d17de033c..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/components/logger/weights_and_biases.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -from typing import TYPE_CHECKING, Dict, List, Optional - -from lightning.app import LightningFlow - -if TYPE_CHECKING: - import wandb - - -class WeightsAndBiases(LightningFlow): - def __init__(self, username: str, project_name: str, run_id: str, api_key: Optional[str] = None) -> None: - super().__init__() - self.username = username - self.project_name = project_name - self.run_id = run_id - self._api_key = api_key - self._run: Optional[wandb.Run] = None - - def run(self) -> None: - if self._run is not None: - return - - if self._api_key: - os.environ["WANDB_API_KEY"] = self._api_key - - import wandb - - self._run = wandb.init(project=self.project_name, id=self.run_id, entity=self.username) - - def configure_layout(self) -> List[Dict[str, str]]: - if self._run is not None: - return [{"name": "Training Logs", "content": self._run.get_url()}] - return [] diff --git a/src/lightning/app/cli/pl-app-template/core/components/script_runner/__init__.py b/src/lightning/app/cli/pl-app-template/core/components/script_runner/__init__.py deleted file mode 100644 index b74bcabd5fbd7..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/components/script_runner/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from core.components.script_runner.script_runner import ScriptRunner # noqa: F401 diff --git a/src/lightning/app/cli/pl-app-template/core/components/script_runner/script_runner.py b/src/lightning/app/cli/pl-app-template/core/components/script_runner/script_runner.py deleted file mode 100644 index 0c2f09e372237..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/components/script_runner/script_runner.py +++ /dev/null @@ -1,76 +0,0 @@ -import sys -import traceback -from typing import Any, Dict, List, Optional, Tuple - -from lightning.app.components.python import TracerPythonScript -from lightning.app.storage.path import Path -from lightning.app.utilities.packaging.build_config import BuildConfig, load_requirements -from lightning.app.utilities.tracer import Tracer - - -class ScriptRunner(TracerPythonScript): - """The ScriptRunner executes the script using ``runpy`` and also patches the Trainer methods to inject additional - code.""" - - def __init__(self, root_path: str, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, cloud_build_config=self._get_build_config(root_path), **kwargs) - self.root_path = root_path - self.exception_message: str = "" - self.trainer_progress: dict = {} - self.trainer_state: dict = {} - self.trainer_hparams: dict = {} - self.model_hparams: dict = {} - self.log_dir: Optional[Path] = None - self.checkpoint_dir: Optional[Path] = None - self.logger_metadatas: List[Dict[str, str]] = [] - - def configure_tracer(self) -> Tracer: - from lightning.pytorch import Trainer - - from core.callbacks import PLAppArtifactsTracker, PLAppProgressTracker, PLAppSummary, PLAppTrainerStateTracker - - tracer = Tracer() - trainer_artifacts_tracker = PLAppArtifactsTracker(work=self) - trainer_state_tracker = PLAppTrainerStateTracker(work=self) - progress_tracker = PLAppProgressTracker(work=self) - summary = PLAppSummary(work=self) - - def pre_trainer_init(_, *args: Any, **kwargs: Any) -> Tuple[Dict, Tuple[Any, ...], Dict[str, Any]]: - kwargs.setdefault("callbacks", []) - kwargs["callbacks"].extend([ - trainer_artifacts_tracker, - trainer_state_tracker, - progress_tracker, - summary, - ]) - return {}, args, kwargs - - tracer.add_traced(Trainer, "__init__", pre_fn=pre_trainer_init) - return tracer - - def run(self) -> None: - self.exception_message = "" - # We need to set the module path both in sys.path and the PYTHONPATH env variable. - # The former is for the current process which is already running, and the env variable is needed in case - # the script launches subprocesses - sys.path.insert(0, self.root_path) - self.env["PYTHONPATH"] = self.root_path - super().run() - - def on_exception(self, exception: BaseException) -> None: - self.exception_message = traceback.format_exc() - super().on_exception(exception) - - @staticmethod - def _get_build_config(root_path: str) -> Optional[BuildConfig]: - # These are the requirements for the script runner itself - requirements = [ - "protobuf<4.21.0", - "pytorch-lightning<=1.6.3", - "pydantic<=1.9.0", - ] - if Path(root_path, "requirements.txt").exists(): - # Requirements from the user's code folder - requirements.extend(load_requirements(root_path, file_name="requirements.txt")) - - return BuildConfig(requirements=requirements) diff --git a/src/lightning/app/cli/pl-app-template/core/state.py b/src/lightning/app/cli/pl-app-template/core/state.py deleted file mode 100644 index 80a9f3d4e0619..0000000000000 --- a/src/lightning/app/cli/pl-app-template/core/state.py +++ /dev/null @@ -1,45 +0,0 @@ -from typing import Dict, Optional, Union - -from pydantic import BaseModel, Field - - -class FitProgress(BaseModel): - current_epoch: int = 0 - train_batch_idx: int = 0 - total_train_batches: int = 0 - val_dataloader_idx: int = 0 - val_batch_idx: int = 0 - total_val_batches: int = 0 - global_step: int = 0 - estimated_stepping_batches: int = 0 - - -class ValidateProgress(BaseModel): - dataloader_idx: int = 0 - val_batch_idx: int = 0 - total_val_batches: int = 0 - - -class TestProgress(BaseModel): - dataloader_idx: int = 0 - test_batch_idx: int = 0 - total_test_batches: int = 0 - - -class PredictProgress(BaseModel): - dataloader_idx: int = 0 - predict_batch_idx: int = 0 - total_predict_batches: int = 0 - - -class ProgressBarState(BaseModel): - fit: FitProgress = Field(default_factory=FitProgress) - val: ValidateProgress = Field(alias="validate", default_factory=ValidateProgress) - test: TestProgress = Field(default_factory=TestProgress) - predict: PredictProgress = Field(default_factory=PredictProgress) - metrics: Dict[str, Union[float, str]] = {} - - -class TrainerState(BaseModel): - fn: Optional[str] = None - stage: Optional[str] = None diff --git a/src/lightning/app/cli/pl-app-template/setup.py b/src/lightning/app/cli/pl-app-template/setup.py deleted file mode 100644 index dc223931779a2..0000000000000 --- a/src/lightning/app/cli/pl-app-template/setup.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -from typing import List - -from setuptools import find_packages, setup - -_PROJECT_ROOT = os.path.dirname(__file__) - - -def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#") -> List[str]: - """Load requirements from a file.""" - with open(os.path.join(path_dir, file_name)) as file: - lines = [ln.strip() for ln in file.readlines()] - reqs = [] - for ln in lines: - # filer all comments - if comment_char in ln: - ln = ln[: ln.index(comment_char)].strip() - # skip directly installed dependencies - if ln.startswith("http"): - continue - # skip index url - if ln.startswith("--extra-index-url"): - continue - if ln: # if requirement is not empty - reqs.append(ln) - return reqs - - -setup( - name="{{ app_name }}", - version="0.0.1", - packages=find_packages(exclude=["ui"]), - python_requires=">=3.8", -) diff --git a/src/lightning/app/cli/pl-app-template/tests/__init__.py b/src/lightning/app/cli/pl-app-template/tests/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/pl-app-template/tests/core/__init__.py b/src/lightning/app/cli/pl-app-template/tests/core/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning/app/cli/pl-app-template/tests/core/test_callbacks.py b/src/lightning/app/cli/pl-app-template/tests/core/test_callbacks.py deleted file mode 100644 index a211da35e6a90..0000000000000 --- a/src/lightning/app/cli/pl-app-template/tests/core/test_callbacks.py +++ /dev/null @@ -1,68 +0,0 @@ -import os.path -from unittest.mock import Mock - -import pytest -from core.callbacks import PLAppArtifactsTracker, PLAppProgressTracker, PLAppSummary -from core.components.script_runner import ScriptRunner -from lightning.app.storage.path import Path -from lightning.pytorch import LightningModule, Trainer -from lightning.pytorch.loggers import TensorBoardLogger - - -@pytest.mark.parametrize("rank", [0, 1]) -def test_progress_tracker_enabled(rank): - trainer = Mock() - trainer.global_rank = rank - trainer.is_global_zero = rank == 0 - work = Mock() - tracker = PLAppProgressTracker(work) - assert not tracker.is_enabled - tracker.setup(trainer, Mock(), Mock()) - assert tracker.is_enabled == trainer.is_global_zero - - -def test_summary_callback_tracks_hyperparameters(): - class ModelWithParameters(LightningModule): - def __init__(self, float_arg=0.1, int_arg=5, bool_arg=True, string_arg="string"): - super().__init__() - self.save_hyperparameters() - - model = ModelWithParameters() - work = Mock() - summary = PLAppSummary(work) - trainer = Trainer(max_epochs=22, callbacks=[summary]) # this triggers the `Callback.on_init_end` hook - summary.setup(trainer, model) - assert work.model_hparams == { - "float_arg": "0.1", - "int_arg": "5", - "bool_arg": "True", - "string_arg": "string", - } - - assert work.trainer_hparams["max_epochs"] == "22" - assert work.trainer_hparams["logger"] == "True" - assert "ModelCheckpoint" in work.trainer_hparams["callbacks"] - assert "PLAppSummary" in work.trainer_hparams["callbacks"] - - -def test_artifacts_tracker(tmpdir): - work = ScriptRunner(root_path=os.path.dirname(__file__), script_path=__file__) - tracker = PLAppArtifactsTracker(work=work) - trainer = Mock() - - trainer.loggers = [] - trainer.default_root_dir = "default_root_dir" - tracker.setup(trainer=trainer, pl_module=Mock()) - assert work.log_dir == Path("default_root_dir") - assert not work.logger_metadatas - - trainer.loggers = [TensorBoardLogger(save_dir=tmpdir)] - trainer.logger = trainer.loggers[0] - tracker.setup(trainer=trainer, pl_module=Mock()) - assert work.log_dir == Path(tmpdir / "lightning_logs" / "version_0") - assert len(work.logger_metadatas) == 1 - assert work.logger_metadatas[0] == {"class_name": "TensorBoardLogger"} - - # call setup a second time and the metadata length should not change - tracker.setup(trainer=trainer, pl_module=Mock()) - assert len(work.logger_metadatas) == 1 diff --git a/src/lightning/app/cli/pl-app-template/tests/test_app.py b/src/lightning/app/cli/pl-app-template/tests/test_app.py deleted file mode 100644 index 3fc14bfcdbf69..0000000000000 --- a/src/lightning/app/cli/pl-app-template/tests/test_app.py +++ /dev/null @@ -1,14 +0,0 @@ -import pytest - - -@pytest.mark.skip() -def test_is_running_in_cloud(monkeypatch): - from app import Main - - monkeypatch.setenv("LIGHTNING_CLOUD_APP_ID", "anything") - app = Main() - assert app.running_in_cloud - - monkeypatch.delenv("LIGHTNING_CLOUD_APP_ID", raising=False) - app = Main() - assert not app.running_in_cloud diff --git a/src/lightning/app/cli/pl-app-template/ui/.gitignore b/src/lightning/app/cli/pl-app-template/ui/.gitignore deleted file mode 100644 index 6c2d44cd3ba13..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.js - -# testing -/coverage - -# production -/build - -# misc -.DS_Store - -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -/cypress/videos -/cypress/screenshots -/cypress/downloads - -.eslintcache diff --git a/src/lightning/app/cli/pl-app-template/ui/.prettierignore b/src/lightning/app/cli/pl-app-template/ui/.prettierignore deleted file mode 100644 index 2ea70f096046d..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/.prettierignore +++ /dev/null @@ -1,3 +0,0 @@ -resources -build -node_modules diff --git a/src/lightning/app/cli/pl-app-template/ui/.prettierrc b/src/lightning/app/cli/pl-app-template/ui/.prettierrc deleted file mode 100644 index cad1459af3548..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/.prettierrc +++ /dev/null @@ -1,24 +0,0 @@ -{ - "jsxSingleQuote": false, - "arrowParens": "avoid", - "tabWidth": 2, - "useTabs": false, - "printWidth": 119, - "singleQuote": false, - "semi": true, - "endOfLine": "lf", - "proseWrap": "always", - "bracketSameLine": true, - "quoteProps": "consistent", - "trailingComma": "all", - "bracketSpacing": true, - "importOrder": [ - "^react$", - "", - "^(components|hooks|resources|utils|lightning-.*)", - "^tests", - "^[./]" - ], - "importOrderSeparation": true, - "importOrderSortSpecifiers": true -} diff --git a/src/lightning/app/cli/pl-app-template/ui/craco.config.js b/src/lightning/app/cli/pl-app-template/ui/craco.config.js deleted file mode 100644 index 979d08985ff19..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/craco.config.js +++ /dev/null @@ -1,29 +0,0 @@ -const path = require("path"); -const fs = require("fs"); -const cracoBabelLoader = require("craco-babel-loader"); - -// manage relative paths to packages -const appDirectory = fs.realpathSync(process.cwd()); -const resolvePackage = relativePath => path.resolve(appDirectory, relativePath); - -module.exports = { - devServer: { - // When launching `yarn start dev`, write the files to the build folder too - devMiddleware: { writeToDisk: true }, - }, - webpack: { - configure: { - output: { - publicPath: "./", - }, - }, - }, - plugins: [ - { - plugin: cracoBabelLoader, - options: { - includes: [resolvePackage("node_modules/lightning-ui")], - }, - }, - ], -}; diff --git a/src/lightning/app/cli/pl-app-template/ui/package.json b/src/lightning/app/cli/pl-app-template/ui/package.json deleted file mode 100644 index 690bd85b6498c..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/package.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "name": "pytorch-lightning-app", - "version": "0.1.0", - "private": true, - "dependencies": { - "@emotion/react": "^11.7.1", - "@emotion/styled": "^11.6.0", - "@mui/icons-material": "^5.6.2", - "@mui/lab": "^5.0.0-alpha.64", - "@mui/material": "^5.2.7", - "@reduxjs/toolkit": "^1.8.0", - "@stripe/stripe-js": "^1.29.0", - "axios": "^0.25.0", - "boring-avatars": "^1.6.3", - "filter-material-ui": "2.7.0", - "fontfaceobserver": "^2.1.0", - "lightning-ui": "git+ssh://git@github.com/gridai/lightning-ui.git#35f4124cc8a16a313174fe63ec82cb74af388c6b", - "lodash": "^4.17.21", - "notistack": "^2.0.4", - "query-string": "^7.1.0", - "react": "^17.0.2", - "react-dom": "^17.0.2", - "react-github-btn": "^1.2.1", - "react-hook-form": "^7.27.1", - "react-query": "^3.34.7", - "react-router-dom": "^6.2.1", - "react-scripts": "5.0.0", - "react-spring": "^9.4.4", - "react-table": "^7.7.0", - "rxjs": "^7.5.2", - "typescript": "^4.4.2", - "use-debounce": "^7.0.1", - "web-vitals": "^2.1.0", - "xterm": "^4.18.0", - "xterm-addon-fit": "^0.5.0", - "xterm-addon-search": "^0.8.2" - }, - "scripts": { - "start": "craco start", - "build": "craco build", - "lint": "eslint --cache --max-warnings=0 . && prettier -c .", - "lint:fix": "eslint --cache --max-warnings=0 . --fix && prettier -w .", - "eject": "react-scripts eject" - }, - "lint-staged": { - "**/*": "prettier --write --ignore-unknown" - }, - "eslintConfig": { - "extends": [ - "react-app" - ], - "ignorePatterns": [ - "node_modules/**", - "build/**" - ], - "rules": { - "react/jsx-sort-props": [ - "off", - { - "callbacksLast": true, - "ignoreCase": true, - "noSortAlphabetically": false, - "reservedFirst": true, - "shorthandFirst": true - } - ], - "react/jsx-pascal-case": "warn" - } - }, - "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] - }, - "devDependencies": { - "@craco/craco": "^6.4.3", - "@trivago/prettier-plugin-sort-imports": "^3.1.1", - "@types/fontfaceobserver": "^2.1.0", - "@types/lodash": "^4.14.182", - "@types/node": "^16.7.13", - "@types/react": "^17.0.20", - "@types/react-dom": "^17.0.9", - "@types/react-table": "^7.7.9", - "craco-babel-loader": "^1.0.3", - "lint-staged": "^12.3.2", - "prettier": "2.5.1" - } -} diff --git a/src/lightning/app/cli/pl-app-template/ui/public/favicon.svg b/src/lightning/app/cli/pl-app-template/ui/public/favicon.svg deleted file mode 100644 index 94a65989d0b4b..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/public/favicon.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/src/lightning/app/cli/pl-app-template/ui/public/index.html b/src/lightning/app/cli/pl-app-template/ui/public/index.html deleted file mode 100644 index 0f2384212e1d3..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/public/index.html +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - - - - PyTorch Lightning App - - - - - - -
- - - diff --git a/src/lightning/app/cli/pl-app-template/ui/public/manifest.json b/src/lightning/app/cli/pl-app-template/ui/public/manifest.json deleted file mode 100644 index 4a97d68d6b7ba..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/public/manifest.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "short_name": "PL App", - "name": "PyTorch Lightning App", - "icons": [ - { - "src": "favicon.svg", - "sizes": "512x512 192x192 64x64 32x32 24x24 16x16", - "type": "image/svg+xml" - } - ], - "start_url": ".", - "display": "standalone", - "theme_color": "#000000", - "background_color": "#ffffff" -} diff --git a/src/lightning/app/cli/pl-app-template/ui/public/robots.txt b/src/lightning/app/cli/pl-app-template/ui/public/robots.txt deleted file mode 100644 index e9e57dc4d41b9..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/public/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -# https://www.robotstxt.org/robotstxt.html -User-agent: * -Disallow: diff --git a/src/lightning/app/cli/pl-app-template/ui/src/App.tsx b/src/lightning/app/cli/pl-app-template/ui/src/App.tsx deleted file mode 100644 index 717984216f298..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/src/App.tsx +++ /dev/null @@ -1,126 +0,0 @@ -import { useEffect } from "react"; - -import { QueryClient, QueryClientProvider } from "react-query"; -import { BrowserRouter } from "react-router-dom"; - -import ErrorPanel from "components/ErrorPanel"; -import HyperparameterSummary from "components/HyperparameterSummary"; -import Launcher from "components/Launcher"; -import ProgressBarGroup from "components/ProgressBarGroup"; -import { - Breadcrumbs, - Card, - CardContent, - CardHeader, - Grid, - SnackbarProvider, - Stack, - useSnackbar, -} from "lightning-ui/src/design-system/components"; -import ThemeProvider from "lightning-ui/src/design-system/theme"; - -import ExecutionSummary from "./components/ExecutionSummary"; -import { useLightningState } from "./hooks/useLightningState"; - -const queryClient = new QueryClient(); - -function AppContainer() { - const { lightningState } = useLightningState(); - - const trainer_progress = lightningState?.flows.script_orchestrator.works.script_runner?.vars.trainer_progress; - const trainer_state = lightningState?.flows.script_orchestrator.works.script_runner?.vars.trainer_state; - const trainer_hparams = lightningState?.flows.script_orchestrator.works.script_runner?.vars.trainer_hparams; - const model_hparams = lightningState?.flows.script_orchestrator.works.script_runner?.vars.model_hparams; - - const script_running = lightningState?.flows.script_orchestrator.vars.running; - const script_succeeded = lightningState?.flows.script_orchestrator.vars.succeeded; - const script_failed = lightningState?.flows.script_orchestrator.vars.failed; - const start_triggered = lightningState?.flows.script_orchestrator.vars.triggered; - const script_path = lightningState?.flows.script_orchestrator.vars.script_path; - const running_in_cloud = lightningState?.vars.running_in_cloud; - - const breadCrumbItems = [ - { title: "Users", href: "url/to/href/1" }, - { title: "adrian", href: "url/to/href/2" }, - { title: "projects", href: "url/to/href/3" }, - { title: "app_name", href: "url/to/href/4" }, - { title: "source", href: "url/to/href/5" }, - { title: "train.py", href: "url/to/href/6" }, - ]; - - const { enqueueSnackbar } = useSnackbar(); - const exception_message = lightningState?.flows.script_orchestrator.works.script_runner?.vars?.exception_message; - useEffect(() => { - if (exception_message) { - enqueueSnackbar({ - title: "The script failed to complete", - severity: "error", - children: "See the error message", - }); - } - }, [exception_message]); - - return ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ); -} - -function App() { - return ( - - - - - - - - - - ); -} - -export default App; diff --git a/src/lightning/app/cli/pl-app-template/ui/src/components/EnvironmentConfigurator.tsx b/src/lightning/app/cli/pl-app-template/ui/src/components/EnvironmentConfigurator.tsx deleted file mode 100644 index 2d26f86ad9965..0000000000000 --- a/src/lightning/app/cli/pl-app-template/ui/src/components/EnvironmentConfigurator.tsx +++ /dev/null @@ -1,67 +0,0 @@ -import { Button, Stack, TextField } from "lightning-ui/src/design-system/components"; - -interface Data { - [key: string]: string; -} - -export function data2dict(data: Data[]) { - var dict: Data = {}; - for (var i = 0; i < data.length; i++) { - if (data[i]["name"] === "") { - continue; - } - dict[data[i]["name"]] = data[i]["value"]; - } - return dict; -} - -export default function EnvironmentConfigurator(props: any) { - const data: Data[] = props.data; - const setData = props.setData; - const addItemAllowed = data[data.length - 1].name.length > 0; - - const onItemAdd = () => { - setData([...data, { name: "", value: "" }]); - }; - - const onItemChange = (fieldName: string, index: number, text: any) => { - let newData = [...data]; - - text = text.trim(); - if (fieldName == "name") { - text = text.replace(/[^0-9a-zA-Z_]+/gi, "").toUpperCase(); - } - - newData[index][fieldName] = text; - setData(newData); - }; - - return ( - - {data.map((entry, index) => ( - - onItemChange("name", index, e)} - placeholder="KEY" - size="medium" - statusText="" - type="text" - value={entry.name || ""} - /> - onItemChange("value", index, e)} - placeholder="VALUE" - size="medium" - statusText="" - type="text" - value={entry.value || ""} - /> - - ))} - -