CI: abstract package actions (#48)

Co-authored-by: Akihiro Nitta <nitta@akihironitta.com>

.github/actions/pkg-create/action.yml

@@ -0,0 +1,51 @@
+name: Create and check package
+description: building, checking the package
+
+inputs:
+ artifact-name:
+ description: 'Unique name for collecting artifacts'
+ required: false
+ default: ""
+
+runs:
+ using: "composite"
+ steps:
+
+ - name: Prepare dev. env
+ run: pip install "twine==4.0.2" setuptools wheel
+ shell: bash
+
+ - name: Source check
+ run: python setup.py check --metadata --strict
+ shell: bash
+
+ - name: Create package 📦
+ # python setup.py clean
+ run: python setup.py sdist bdist_wheel
+ shell: bash
+
+ - name: Check package 📦
+ working-directory: dist
+ run: |
+ ls -lh .
+ twine check *
+ shell: bash
+
+ - name: Download 📥 previous packages
+ if: ${{ inputs.artifact-name != '' }}
+ uses: actions/download-artifact@v3
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: pypi
+
+ - name: copy/export pkg
+ if: ${{ inputs.artifact-name != '' }}
+ run: cp dist/* pypi/
+ shell: bash
+
+ - name: Upload 📤 to the share store
+ if: ${{ inputs.artifact-name != '' }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: pypi

.github/actions/pkg-install/action.yml

@@ -0,0 +1,53 @@
+name: Install and check package
+description: installing and validationg the package
+
+inputs:
+ artifact-name:
+ description: 'Unique name for collecting artifacts'
+ required: false
+ default: ""
+ import-name:
+ description: 'Import name to test with after installation'
+ required: true
+ pip-flags:
+ description: additional pil install flags
+ required: false
+ default: ""
+
+runs:
+ using: "composite"
+ steps:
+ - name: Download 📥 all packages
+ if: ${{ inputs.artifact-name != '' }}
+ uses: actions/download-artifact@v3
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: pypi
+ - name: show packages
+ working-directory: pypi
+ run: ls -lh
+ shell: bash
+
+ - name: Install package (wheel)
+ working-directory: pypi/
+ run: |
+ # TODO: reset env / conside add as conda
+ pip install *.whl ${{ inputs.install-flags }}
+ python -c "import ${{ inputs.import-name }} as pkg; print(f'version: {pkg.__version__}')"
+ pip list
+ shell: bash
+
+ - name: Uninstall all
+ run: |
+ pip freeze > _reqs.txt
+ pip uninstall -y -r _reqs.txt
+ shell: bash
+
+ - name: Install package (archive)
+ working-directory: pypi/
+ run: |
+ # TODO: reset env / conside add as conda
+ pip install *.tar.gz ${{ inputs.install-flags }}
+ python -c "import ${{ inputs.import-name }} as pkg; print(f'version: {pkg.__version__}')"
+ pip list
+ shell: bash

.github/workflows/check-docs.yml

@@ -32,7 +32,6 @@ jobs:
 
  docs-test:
  runs-on: ubuntu-20.04
-
  steps:
  - name: Checkout 🛎
  uses: actions/checkout@v3
@@ -65,7 +64,6 @@ jobs:
 
  docs-html:
  runs-on: ubuntu-20.04
-
  steps:
  - name: Checkout 🛎
  uses: actions/checkout@v3

.github/workflows/check-package.yml

@@ -3,6 +3,10 @@ name: Check package flow
 on:
  workflow_call:
  inputs:
+ actions-ref:
+ description: 'Version of actions, normally the same as workflow'
+ required: true
+ type: string
  artifact-name:
  description: 'Unique name for collecting artifacts'
  required: true
@@ -42,13 +46,12 @@ defaults:
 
 jobs:
 
- init-store:
+ init-store: # todo: remove this wne download will be fault tolerant
  runs-on: ubuntu-20.04
  steps:
  - name: Checkout 🛎️
  uses: actions/checkout@v3
- - run: |
- mkdir dist && touch dist/.placeholder
+ - run: mkdir dist && touch dist/.placeholder
  - name: Upload 📤
  uses: actions/upload-artifact@v3
  with:
@@ -67,38 +70,21 @@ jobs:
  steps:
  - name: Checkout 🛎️
  uses: actions/checkout@v3
- - name: Download 📥 previous packages
- uses: actions/download-artifact@v3
- with:
- name: ${{ inputs.artifact-name }}
- path: pypi
  - name: Set up Python 🐍
  uses: actions/setup-python@v4
  with:
  python-version: 3.8
 
- - name: Check local package
- run: python setup.py check --metadata --strict
-
- - name: Create package
- run: |
- pip install -q -U setuptools wheel
- python setup.py sdist bdist_wheel
- python setup.py clean
-
- - name: Check build package
- run: |
- pip install -q -U twine>=4.0.1
- # check package description
- twine check dist/*
-
- - name: prepare for upload
- run: cp dist/* pypi/
- - name: Upload 📤 to the share store
- uses: actions/upload-artifact@v3
+ - name: Pull re-usable 🤖 actions️
+ uses: actions/checkout@v3
  with:
- name: ${{ inputs.artifact-name }}
- path: pypi
+ ref: ${{ inputs.actions-ref }}
+ path: .cicd
+ repository: Lightning-AI/utilities
+ - name: Create package 📦
+ uses: ./.cicd/.github/actions/pkg-create
+ with:
+ artifact-name: ${{ inputs.artifact-name }}
 
 
  pkg-check:
@@ -109,38 +95,24 @@ jobs:
  matrix: ${{ fromJSON(inputs.testing-matrix) }}
 
  steps:
- - name: Download 📥 all packages
- uses: actions/download-artifact@v3
- with:
- name: ${{ inputs.artifact-name }}
- path: pypi
- - name: show packages
- working-directory: pypi
- run: ls -lh
+ - name: Checkout 🛎️
+ uses: actions/checkout@v3
  - name: Set up Python 🐍 ${{ matrix.python-version }}
  uses: actions/setup-python@v4
  with:
  python-version: ${{ matrix.python-version }}
 
- - name: Install package (wheel)
- working-directory: pypi
- run: |
- # TODO: reset env / conside add as conda
- pip install *.whl ${{ inputs.install-flags }}
- python -c "import ${{ inputs.import-name }} as pkg; print(f'version: {pkg.__version__}')"
- pip list
-
- - name: Uninstall all
- run: |
- pip freeze > _reqs.txt
- pip uninstall -y -r _reqs.txt
-
- - name: Install package (archive)
- working-directory: pypi
- run: |
- # TODO: reset env / conside add as conda
- pip install *.tar.gz ${{ inputs.install-flags }}
- python -c "import ${{ inputs.import-name }} as pkg; print(f'version: {pkg.__version__}')"
- pip list
+ - name: Pull re-usable 🤖 actions️
+ uses: actions/checkout@v3
+ with:
+ ref: ${{ inputs.actions-ref }}
+ path: .cicd
+ repository: Lightning-AI/utilities
+ - name: Installing package 📦
+ uses: ./.cicd/.github/actions/pkg-install
+ with:
+ artifact-name: ${{ inputs.artifact-name }}
+ import-name: ${{ inputs.import-name }}
+ pip-flags: ${{ inputs.install-flags }}
 
  # TODO: add run doctests

.github/workflows/ci-use-checks.yml

@@ -24,6 +24,7 @@ jobs:
  check-package:
  uses: ./.github/workflows/check-package.yml
  with:
+ actions-ref: ${{ github.sha }} # use local version
  artifact-name: dist-packages-${{ github.sha }}
  import-name: "lightning_utilities"
 

.github/workflows/publish-pypi.yml

@@ -3,14 +3,17 @@ name: PyPI Release
 # https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on: # Trigger the workflow on push or pull request, but only for the master branch
  push:
- tags:
- - 'v?[0-9]+.[0-9]+.[0-9]+'
+ branches: [main, "release/*"]
+ tags: ['v?[0-9]+.[0-9]+.[0-9]+']
+ release:
+ types: [published]
 
 defaults:
  run:
  shell: bash
 
 jobs:
+
  # based on https://github.com/pypa/gh-action-pypi-publish
  build-package:
  runs-on: ubuntu-20.04
@@ -21,28 +24,23 @@ jobs:
  - name: Set up Python 🐍
  uses: actions/setup-python@v4
  with:
- python-version: 3.8
-
- - name: Install dependencies
- run: >-
- python -m pip install --user --upgrade setuptools wheel
-
- - name: Build packages
- run: |
- python setup.py sdist bdist_wheel
- ls -lh dist/
+ python-version: 3.8
 
- - name: Upload 📤 artifact
+ - name: Create package 📦
+ uses: ./.github/actions/pkg-create
+ - name: Upload 📤 to the share store
+ if: ${{ inputs.artifact-name != '' }}
  uses: actions/upload-artifact@v3
  with:
  name: pypi-packages-${{ github.sha }}
  path: dist
 
 
  upload-package:
+ needs: build-package
+ if: github.event_name == 'release'
  timeout-minutes: 5
  runs-on: ubuntu-20.04
- needs: build-package
  steps:
  - name: Checkout 🛎️
  uses: actions/checkout@v3
@@ -62,10 +60,21 @@ jobs:
  repo-token: ${{ secrets.GITHUB_TOKEN }}
 
 
+ delay-publish:
+ needs: build-package
+ runs-on: ubuntu-20.04
+ steps:
+ - name: Delay ⏰ releasing
+ uses: juliangruber/sleep-action@v1
+ with:
+ time: 5m
+
+
  publish-package:
+ needs: delay-publish
+ if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
  runs-on: ubuntu-20.04
- timeout-minutes: 10
- needs: build-package
+ timeout-minutes: 5
  steps:
  - name: Checkout 🛎️
  uses: actions/checkout@v3
@@ -77,12 +86,6 @@ jobs:
  - name: local 🗃️ files
  run: ls -lh dist/
 
- - name: Delay releasing
- if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
- uses: juliangruber/sleep-action@v1
- with:
- time: 5m
-
  # We do this, since failures on test.pypi aren't that bad
  - name: Publish to Test PyPI
  if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'