CI: signal lai build

.github/actions/pkg-publish/action.yml

@@ -5,9 +5,14 @@ inputs:
   pkg-pattern:
     description: what file pattern is searched in folder, so for example `*_app*`
     required: true
+  pypi-test-token:
+    description: login token for PyPI
+    default: ''
+    required: false
   pypi-token:
     description: login token for PyPI
-    required: true
+    default: ''
+    required: false
 
 runs:
   using: "composite"
@@ -20,16 +25,19 @@ runs:
     shell: bash
 
   # We do this, since failures on test.pypi aren't that bad
-  #- name: Publish to Test PyPI
-  #  uses: pypa/gh-action-pypi-publish@v1.5.1
-  #  with:
-  #    user: __token__
-  #    password: ${{ secrets.test_pypi_token_lai }}
-  #    repository_url: https://test.pypi.org/legacy/
-  #    verbose: true
+  - name: Publish to Test PyPI
+    uses: pypa/gh-action-pypi-publish@v1.5.1
+    if: inputs.pypi-test-token != ''
+    with:
+      user: __token__
+      password: ${{ secrets.test_pypi_token_lai }}
+      repository_url: https://test.pypi.org/legacy/
+      packages_dir: pypi/
+      verbose: true
 
   - name: Publish distribution 📦 to PyPI
     uses: pypa/gh-action-pypi-publish@v1.5.1
+    if: inputs.pypi-token != ''
     with:
       user: __token__
       password: ${{ inputs.pypi-token }}

.github/workflows/legacy-checkpoints.yml

@@ -41,11 +41,15 @@ on:
       AWS_SECRET_KEY_ID:
         required: true
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   create-legacy-ckpts:
     runs-on: ubuntu-20.04
     outputs:
-      pl-version: ${{ steps.decide-pl-version.outputs.pl-version }}
+      pl-version: ${{ steps.decide-version.outputs.pl-version }}
     defaults:
       run:
         working-directory: tests/legacy
@@ -69,16 +73,24 @@ jobs:
       env:
         PACKAGE_NAME: pytorch
       run: |
-        pip install -e . -f https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html
         pip list
       if: inputs.pl_version == ''
 
+    - name: Install PL version
+      run: |
+        pip install "pytorch-lightning==${{ inputs.pl_version }}" \
+          -f https://download.pytorch.org/whl/cpu/torch_stable.html \
+          --extra-index-url https://test.pypi.org/simple/
+        pip list
+      if: inputs.pl_version != ''
+
     - name: Pull legacy checkpoints
       working-directory: ./
       run: bash .actions/pull_legacy_checkpoints.sh
 
     - name: Decide PL version to create a PR with
-      id: decide-pl-version
+      id: decide-version
       run: |
         python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT
 

.github/workflows/release-pypi.yml

@@ -7,6 +7,10 @@ on:
   release:
     types: [published]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   init:
     runs-on: ubuntu-20.04
@@ -24,8 +28,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: true
-      # run sequential
-      max-parallel: 1
+      max-parallel: 1  # run sequential to prevent download/upload collisions
       matrix:
         pkg-name: ["lightning", "app", "lite", "pytorch"]
     steps:
@@ -70,7 +73,69 @@ jobs:
         files: 'dist/*'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-  publish-packages:
+  release-version:
+    runs-on: ubuntu-20.04
+    outputs:
+      tag: ${{ steps.lai-package.outputs.version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: install Package
+      env:
+        PACKAGE_NAME: "lightning"
+      run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html
+    - name: package Version
+      id: lai-package
+      run: python -c "import lightning as L; print(f'version={L.__version__}')" >> $GITHUB_OUTPUT
+
+  signaling:
+    runs-on: ubuntu-20.04
+    needs: [release-version, pre-publish-packages]
+    env:
+      TAG: ${{ needs.release-version.outputs.tag }}
+    steps:
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+    - uses: actions/checkout@v3
+      with:
+        repository: gridai/base-images
+        token: ${{ secrets.PAT_GHOST }}
+        ref: main
+    - uses: fregante/setup-git-token@v1
+      with:
+        token: ${{ secrets.PAT_GHOST }}
+        name: PL Ghost
+        email: pl-github@grid.ai
+    - name: Update lightning version
+      run: |
+        import json, os
+        with open("versions.json") as fo:
+            vers = json.load(fo)
+        vers["lightning_version"] = os.getenv('TAG')
+        with open("versions.json", "w") as fw:
+            json.dump(vers, fw)
+      shell: python
+    - name: GIT Commit
+      run: |
+        git add versions.json
+        git commit -m "bumping lightning version -> ${TAG}"
+        cat versions.json
+    - name: GIT Push
+      run: |
+        git status
+        git push
+
+  waiting:
+    # TODO: replace with back signal from build images/ loop checking for a specific branch?
+    runs-on: ubuntu-20.04
+    needs: signaling
+    steps:
+    - name: Delay releasing
+      uses: juliangruber/sleep-action@v1
+      with:
+        time: 30m
+
+  pre-publish-packages:
     runs-on: ubuntu-20.04
     needs: build-packages
     if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
@@ -82,38 +147,59 @@ jobs:
         path: dist
     - run: ls -lh dist/
     - run: mkdir pypi/
-
-    - name: Delay releasing
-      uses: juliangruber/sleep-action@v1
+    - uses: ./.github/actions/pkg-publish
       with:
-        time: 10m
+        pkg-pattern: "*app*"
+        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_APP }}
+    - uses: ./.github/actions/pkg-publish
+      with:
+        pkg-pattern: "*lite*"
+        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LITE }}
+    - uses: ./.github/actions/pkg-publish
+      with:
+        pkg-pattern: "*pytorch*"
+        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_PYTORCH }}
+    - uses: ./.github/actions/pkg-publish
+      with:
+        pkg-pattern: "*"
+        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LAI }}
 
+  publish-packages:
+    runs-on: ubuntu-20.04
+    needs: waiting
+    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist-packages-${{ github.sha }}
+        path: dist
+    - run: ls -lh dist/
+    - run: mkdir pypi/
     - uses: ./.github/actions/pkg-publish
       with:
         pkg-pattern: "*app*"
         pypi-token: ${{ secrets.PYPI_TOKEN_APP }}
-
     - uses: ./.github/actions/pkg-publish
       with:
         pkg-pattern: "*lite*"
         pypi-token: ${{ secrets.PYPI_TOKEN_LITE }}
-
     - uses: ./.github/actions/pkg-publish
       with:
         pkg-pattern: "*pytorch*"
         pypi-token: ${{ secrets.PYPI_TOKEN_PYTORCH }}
-
     - uses: ./.github/actions/pkg-publish
       with:
         pkg-pattern: "*"
         pypi-token: ${{ secrets.PYPI_TOKEN_LAI }}
 
   legacy-checkpoints:
-    needs: publish-packages
+    needs: [release-version, pre-publish-packages]
     uses: ./.github/workflows/legacy-checkpoints.yml
     with:
       push_to_s3: true
       create_pr: true
+      pl_version: ${{ needs.release-version.outputs.tag }}
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_KEY_ID: ${{ secrets.AWS_SECRET_KEY_ID }}