From 74c4498f102b03c0668d981ee9db154fe176dcf3 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 09:44:53 +0100 Subject: [PATCH 01/10] split --- .github/workflows/release-pypi.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index a3d7b85ab9935..bd151226da3fc 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -2,8 +2,8 @@ name: PyPI # https://help.github.com/en/actions/reference/events-that-trigger-workflows on: - push: - branches: [master, "release/*"] + push: {} # FIXME +# branches: [master, "release/*"] release: types: [published] @@ -70,9 +70,18 @@ jobs: files: 'dist/*' repo-token: ${{ secrets.GITHUB_TOKEN }} - publish-packages: + waiting: runs-on: ubuntu-20.04 needs: build-packages + steps: + - name: Delay releasing + uses: juliangruber/sleep-action@v1 + with: + time: 30m + + publish-packages: + runs-on: ubuntu-20.04 + needs: waiting if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' steps: - uses: actions/checkout@v3 @@ -83,26 +92,18 @@ jobs: - run: ls -lh dist/ - run: mkdir pypi/ - - name: Delay releasing - uses: juliangruber/sleep-action@v1 - with: - time: 10m - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*app*" pypi-token: ${{ secrets.PYPI_TOKEN_APP }} - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*lite*" pypi-token: ${{ secrets.PYPI_TOKEN_LITE }} - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*pytorch*" pypi-token: ${{ secrets.PYPI_TOKEN_PYTORCH }} - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*" From b404da1fb53d2b0a081b45a97a9137273d0446b0 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 10:13:40 +0100 Subject: [PATCH 02/10] ... --- .github/workflows/release-pypi.yml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index bd151226da3fc..68c60acce4487 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -70,6 +70,34 @@ jobs: files: 'dist/*' repo-token: ${{ secrets.GITHUB_TOKEN }} + signaling: + runs-on: ubuntu-20.04 + needs: build-packages # TODO: this shall be `upload-packages` + steps: + - uses: actions/checkout@v3 + with: + repository: gridai/base-images + token: ${{ secrets.PAT_GHOST }} + ref: master + - uses: fregante/setup-git-token@v1 + with: + token: ${{ secrets.PAT_GHOST }} + name: PL Ghost + email: pl-github@grid.ai + - run: ls -l + # TODO bump version + - run: | + import json + with open("versions.json") as fo: + vers = json.load(fo) + print(vers) + - name: GIT Commit + run: | + git add versions.json + git status + git commit -m "bumping version TBD" + git push + waiting: runs-on: ubuntu-20.04 needs: build-packages @@ -81,7 +109,7 @@ jobs: publish-packages: runs-on: ubuntu-20.04 - needs: waiting + needs: [waiting, signaling] if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' steps: - uses: actions/checkout@v3 From bf4a8e3c75c27297bfe059adbe2765386d30ffdd Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 12:35:41 +0100 Subject: [PATCH 03/10] ..- --- .github/workflows/release-pypi.yml | 46 ++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 68c60acce4487..f6ec5500dee51 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -7,6 +7,10 @@ on: release: types: [published] +defaults: + run: + shell: bash + jobs: init: runs-on: ubuntu-20.04 @@ -24,8 +28,7 @@ jobs: runs-on: ubuntu-20.04 strategy: fail-fast: true - # run sequential - max-parallel: 1 + max-parallel: 1 # run sequential to prevent download/upload collisions matrix: pkg-name: ["lightning", "app", "lite", "pytorch"] steps: @@ -70,32 +73,57 @@ jobs: files: 'dist/*' repo-token: ${{ secrets.GITHUB_TOKEN }} + release-version: + runs-on: ubuntu-20.04 + outputs: + tag: ${{ steps.lai-package.outputs.version }} + steps: + - uses: actions/checkout@v3 + - name: install Package + env: + PACKAGE_NAME: "lightning" + run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html + - name: package Version + id: lai-package + run: python -c "import lightning as L; print(f'version={L.__version__}')" >> $GITHUB_OUTPUT + signaling: runs-on: ubuntu-20.04 - needs: build-packages # TODO: this shall be `upload-packages` + # TODO: instead of `build-packages` this shall be `upload-packages` + needs: [release-version, build-packages] steps: + - uses: actions/setup-python@v4 + with: + python-version: 3.8 - uses: actions/checkout@v3 with: repository: gridai/base-images token: ${{ secrets.PAT_GHOST }} - ref: master + ref: master # FIXME - uses: fregante/setup-git-token@v1 with: token: ${{ secrets.PAT_GHOST }} name: PL Ghost email: pl-github@grid.ai - - run: ls -l - # TODO bump version - - run: | + - name: Update lightning version + env: + TAG: ${{ needs.release-version.outputs.tag }} + run: | import json with open("versions.json") as fo: vers = json.load(fo) - print(vers) + vers["lightning_version"] = $TAG + with open("versions.json", "w") as fw: + vers = json.dump(vers, fo) + shell: python - name: GIT Commit + env: + TAG: ${{ needs.release-version.outputs.tag }} run: | + cat versions.json git add versions.json git status - git commit -m "bumping version TBD" + git commit -m "bumping lightning version -> $TAG" git push waiting: From 826cbc805b09ff795a0d009b086df63a74ef8faa Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 12:41:41 +0100 Subject: [PATCH 04/10] .-- --- .github/workflows/release-pypi.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index f6ec5500dee51..20c8cf13c0080 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -91,6 +91,8 @@ jobs: runs-on: ubuntu-20.04 # TODO: instead of `build-packages` this shall be `upload-packages` needs: [release-version, build-packages] + env: + TAG: ${{ needs.release-version.outputs.tag }} steps: - uses: actions/setup-python@v4 with: @@ -106,24 +108,20 @@ jobs: name: PL Ghost email: pl-github@grid.ai - name: Update lightning version - env: - TAG: ${{ needs.release-version.outputs.tag }} run: | import json with open("versions.json") as fo: vers = json.load(fo) - vers["lightning_version"] = $TAG + vers["lightning_version"] = "${TAG}" with open("versions.json", "w") as fw: vers = json.dump(vers, fo) shell: python - name: GIT Commit - env: - TAG: ${{ needs.release-version.outputs.tag }} run: | cat versions.json git add versions.json git status - git commit -m "bumping lightning version -> $TAG" + git commit -m "bumping lightning version -> ${TAG}" git push waiting: From 6a65eefe56d4a5d146ad1f74ba8ef4f1c8fc0966 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 13:09:15 +0100 Subject: [PATCH 05/10] --- --- .github/workflows/release-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 20c8cf13c0080..999ccfa71308a 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -114,7 +114,7 @@ jobs: vers = json.load(fo) vers["lightning_version"] = "${TAG}" with open("versions.json", "w") as fw: - vers = json.dump(vers, fo) + json.dump(vers, fw) shell: python - name: GIT Commit run: | From e0ae23b8005beab4bf7c380d61eef10e92607a84 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 13:12:54 +0100 Subject: [PATCH 06/10] --. --- .github/workflows/release-pypi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 999ccfa71308a..b0796e6df1bc2 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -109,10 +109,10 @@ jobs: email: pl-github@grid.ai - name: Update lightning version run: | - import json + import json, os with open("versions.json") as fo: vers = json.load(fo) - vers["lightning_version"] = "${TAG}" + vers["lightning_version"] = os.getenv('TAG') with open("versions.json", "w") as fw: json.dump(vers, fw) shell: python From 7f080ba2a08fb13c6729b9c75ec8776a13327c5e Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 13:23:49 +0100 Subject: [PATCH 07/10] -.. --- .github/workflows/release-pypi.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index b0796e6df1bc2..0cd6a5f1aa2bd 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -120,8 +120,10 @@ jobs: run: | cat versions.json git add versions.json - git status git commit -m "bumping lightning version -> ${TAG}" + - name: GIT Push + run: | + git status git push waiting: From cbe136fc938323e825d53764d5ab32f8326a7035 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 13:33:52 +0100 Subject: [PATCH 08/10] ... --- .github/workflows/release-pypi.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 0cd6a5f1aa2bd..7c6113cedffe8 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -89,8 +89,7 @@ jobs: signaling: runs-on: ubuntu-20.04 - # TODO: instead of `build-packages` this shall be `upload-packages` - needs: [release-version, build-packages] + needs: [release-version, upload-packages] env: TAG: ${{ needs.release-version.outputs.tag }} steps: @@ -118,9 +117,9 @@ jobs: shell: python - name: GIT Commit run: | - cat versions.json git add versions.json git commit -m "bumping lightning version -> ${TAG}" + cat versions.json - name: GIT Push run: | git status From 5cee9ca066b3a3c5253740c3226bb0f0ffca8b10 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 15:29:22 +0100 Subject: [PATCH 09/10] .._ --- .github/actions/pkg-publish/action.yml | 24 +++++++++----- .github/workflows/legacy-checkpoints.yml | 18 +++++++++-- .github/workflows/release-pypi.yml | 40 +++++++++++++++++++++--- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/.github/actions/pkg-publish/action.yml b/.github/actions/pkg-publish/action.yml index c9b202273f68c..b9362784ae6c5 100644 --- a/.github/actions/pkg-publish/action.yml +++ b/.github/actions/pkg-publish/action.yml @@ -5,9 +5,14 @@ inputs: pkg-pattern: description: what file pattern is searched in folder, so for example `*_app*` required: true + pypi-test-token: + description: login token for PyPI + default: '' + required: false pypi-token: description: login token for PyPI - required: true + default: '' + required: false runs: using: "composite" @@ -20,16 +25,19 @@ runs: shell: bash # We do this, since failures on test.pypi aren't that bad - #- name: Publish to Test PyPI - # uses: pypa/gh-action-pypi-publish@v1.5.1 - # with: - # user: __token__ - # password: ${{ secrets.test_pypi_token_lai }} - # repository_url: https://test.pypi.org/legacy/ - # verbose: true + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@v1.5.1 + if: inputs.pypi-test-token != '' + with: + user: __token__ + password: ${{ secrets.test_pypi_token_lai }} + repository_url: https://test.pypi.org/legacy/ + packages_dir: pypi/ + verbose: true - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@v1.5.1 + if: inputs.pypi-token != '' with: user: __token__ password: ${{ inputs.pypi-token }} diff --git a/.github/workflows/legacy-checkpoints.yml b/.github/workflows/legacy-checkpoints.yml index 9fa98cc5b1049..5e08455d22c8f 100644 --- a/.github/workflows/legacy-checkpoints.yml +++ b/.github/workflows/legacy-checkpoints.yml @@ -41,11 +41,15 @@ on: AWS_SECRET_KEY_ID: required: true +defaults: + run: + shell: bash + jobs: create-legacy-ckpts: runs-on: ubuntu-20.04 outputs: - pl-version: ${{ steps.decide-pl-version.outputs.pl-version }} + pl-version: ${{ steps.decide-version.outputs.pl-version }} defaults: run: working-directory: tests/legacy @@ -69,16 +73,24 @@ jobs: env: PACKAGE_NAME: pytorch run: | - pip install -e . -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html pip list if: inputs.pl_version == '' + - name: Install PL version + run: | + pip install "pytorch-lightning==${{ inputs.pl_version }}" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html \ + --extra-index-url https://test.pypi.org/simple/ + pip list + if: inputs.pl_version != '' + - name: Pull legacy checkpoints working-directory: ./ run: bash .actions/pull_legacy_checkpoints.sh - name: Decide PL version to create a PR with - id: decide-pl-version + id: decide-version run: | python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 7c6113cedffe8..4bc2dd26d068e 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -89,7 +89,7 @@ jobs: signaling: runs-on: ubuntu-20.04 - needs: [release-version, upload-packages] + needs: [release-version, pre-publish-packages] env: TAG: ${{ needs.release-version.outputs.tag }} steps: @@ -126,17 +126,18 @@ jobs: git push waiting: + # TODO: replace with back signal from build images/ loop checking for a specific branch? runs-on: ubuntu-20.04 - needs: build-packages + needs: signaling steps: - name: Delay releasing uses: juliangruber/sleep-action@v1 with: time: 30m - publish-packages: + pre-publish-packages: runs-on: ubuntu-20.04 - needs: [waiting, signaling] + needs: build-packages if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' steps: - uses: actions/checkout@v3 @@ -146,7 +147,35 @@ jobs: path: dist - run: ls -lh dist/ - run: mkdir pypi/ + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*app*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_APP }} + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*lite*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LITE }} + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*pytorch*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_PYTORCH }} + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LAI }} + publish-packages: + runs-on: ubuntu-20.04 + needs: waiting + if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: dist-packages-${{ github.sha }} + path: dist + - run: ls -lh dist/ + - run: mkdir pypi/ - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*app*" @@ -165,11 +194,12 @@ jobs: pypi-token: ${{ secrets.PYPI_TOKEN_LAI }} legacy-checkpoints: - needs: publish-packages + needs: [release-version, pre-publish-packages] uses: ./.github/workflows/legacy-checkpoints.yml with: push_to_s3: true create_pr: true + pl_version: ${{ needs.release-version.outputs.tag }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_KEY_ID: ${{ secrets.AWS_SECRET_KEY_ID }} From f959d814226275815cf759705e259e3571ce92ce Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 30 Nov 2022 15:35:37 +0100 Subject: [PATCH 10/10] clean --- .github/workflows/release-pypi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 4bc2dd26d068e..ba7f18341ac01 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -2,8 +2,8 @@ name: PyPI # https://help.github.com/en/actions/reference/events-that-trigger-workflows on: - push: {} # FIXME -# branches: [master, "release/*"] + push: + branches: [master, "release/*"] release: types: [published] @@ -100,7 +100,7 @@ jobs: with: repository: gridai/base-images token: ${{ secrets.PAT_GHOST }} - ref: master # FIXME + ref: main - uses: fregante/setup-git-token@v1 with: token: ${{ secrets.PAT_GHOST }}