Do not publish wheels for all SHAs, and publish to PyPI from Github A…

…ctions. (#19028) This change: 1. Adjusts wheel building to only upload to S3 when requested via a workflow or tag push 2. Adjusts the release job to build and upload wheels to S3, and then also to push them to PyPI. * Note that this does not yet publish our "universal" PEX from the CI job, so that step still happens locally. See the docs changes for a high level overview.
pantsbuild · May 18, 2023 · 41551ad · 41551ad
1 parent d965c95
commit 41551ad
Show file tree

Hide file tree

Showing 7 changed files with 408 additions and 181 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -4,24 +4,286 @@
 
 
 jobs:
-  publish-tag-to-commit-mapping:
+  build_wheels_linux_arm64:
+    container:
+      image: ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf
+    env:
+      PANTS_REMOTE_CACHE_READ: 'false'
+      PANTS_REMOTE_CACHE_WRITE: 'false'
     if: github.repository_owner == 'pantsbuild'
+    name: Build wheels (Linux-ARM64)
+    runs-on:
+    - self-hosted
+    - Linux
+    - ARM64
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 10
+    - name: Configure Git
+      run: git config --global safe.directory "$GITHUB_WORKSPACE"
+    - name: Install rustup
+      run: 'curl --proto ''=https'' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s --
+        -v -y --default-toolchain none
+
+        echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+
+        '
+    - name: Expose Pythons
+      run: 'echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH
+
+        echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH
+
+        echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH
+
+        '
+    - if: github.event_name != 'pull_request'
+      name: Setup toolchain auth
+      run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV
+
+        '
+    - env:
+        PANTS_CONFIG_FILES: +['pants.ci.toml','pants.ci.aarch64.toml']
+      name: Build wheels
+      run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex
+
+
+        USE_PY39=true ./build-support/bin/release.sh build-wheels
+
+        ./build-support/bin/release.sh build-wheels
+
+        USE_PY38=true ./build-support/bin/release.sh build-wheels'
+    - continue-on-error: true
+      if: always()
+      name: Upload pants.log
+      uses: actions/upload-artifact@v3
+      with:
+        name: pants-log-wheels-Linux-ARM64
+        path: .pants.d/pants.log
+    - name
+    - run
+    - env
+    timeout-minutes: 90
+  build_wheels_linux_x86_64:
+    container:
+      image: quay.io/pypa/manylinux2014_x86_64:latest
+    env:
+      PANTS_REMOTE_CACHE_READ: 'false'
+      PANTS_REMOTE_CACHE_WRITE: 'false'
+    if: github.repository_owner == 'pantsbuild'
+    name: Build wheels (Linux-x86_64)
+    runs-on:
+    - ubuntu-20.04
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 10
+    - name: Configure Git
+      run: git config --global safe.directory "$GITHUB_WORKSPACE"
+    - name: Install rustup
+      run: 'curl --proto ''=https'' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s --
+        -v -y --default-toolchain none
+
+        echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+
+        '
+    - name: Expose Pythons
+      run: 'echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH
+
+        echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH
+
+        echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH
+
+        '
+    - if: github.event_name != 'pull_request'
+      name: Setup toolchain auth
+      run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV
+
+        '
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.5
+    - env: {}
+      name: Build wheels
+      run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex
+
+
+        USE_PY39=true ./build-support/bin/release.sh build-wheels
+
+        ./build-support/bin/release.sh build-wheels
+
+        USE_PY38=true ./build-support/bin/release.sh build-wheels'
+    - continue-on-error: true
+      if: always()
+      name: Upload pants.log
+      uses: actions/upload-artifact@v3
+      with:
+        name: pants-log-wheels-Linux-x86_64
+        path: .pants.d/pants.log
+    - name
+    - run
+    - env
+    timeout-minutes: 90
+  build_wheels_macos10_15_x86_64:
+    env:
+      PANTS_REMOTE_CACHE_READ: 'false'
+      PANTS_REMOTE_CACHE_WRITE: 'false'
+    if: github.repository_owner == 'pantsbuild'
+    name: Build wheels (macOS10-15-x86_64)
+    runs-on:
+    - self-hosted
+    - macOS-10.15-X64
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 10
+        ref: ${{ needs.determine_ref.outputs.build-ref }}
+    - name: Cache Rust toolchain
+      uses: actions/cache@v3
+      with:
+        key: macOS10-15-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        path: '~/.rustup/toolchains/1.69.0-*
+
+          ~/.rustup/update-hashes
+
+          ~/.rustup/settings.toml
+
+          '
+    - name: Cache Cargo
+      uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1
+      with:
+        cache-bin: 'false'
+        shared-key: engine
+        workspaces: src/rust/engine
+    - if: github.event_name != 'pull_request'
+      name: Setup toolchain auth
+      run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV
+
+        '
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.5
+    - env:
+        ARCHFLAGS: -arch x86_64
+      name: Build wheels
+      run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex
+
+
+        USE_PY39=true ./build-support/bin/release.sh build-wheels
+
+        ./build-support/bin/release.sh build-wheels
+
+        USE_PY38=true ./build-support/bin/release.sh build-wheels'
+    - continue-on-error: true
+      if: always()
+      name: Upload pants.log
+      uses: actions/upload-artifact@v3
+      with:
+        name: pants-log-wheels-macOS10-15-x86_64
+        path: .pants.d/pants.log
+    - name
+    - run
+    - env
+    timeout-minutes: 90
+  build_wheels_macos11_arm64:
+    env:
+      PANTS_REMOTE_CACHE_READ: 'false'
+      PANTS_REMOTE_CACHE_WRITE: 'false'
+    if: github.repository_owner == 'pantsbuild'
+    name: Build wheels (macOS11-ARM64)
+    runs-on:
+    - self-hosted
+    - macOS-11-ARM64
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 10
+        ref: ${{ needs.determine_ref.outputs.build-ref }}
+    - name: Cache Rust toolchain
+      uses: actions/cache@v3
+      with:
+        key: macOS11-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v2
+        path: '~/.rustup/toolchains/1.69.0-*
+
+          ~/.rustup/update-hashes
+
+          ~/.rustup/settings.toml
+
+          '
+    - name: Cache Cargo
+      uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1
+      with:
+        cache-bin: 'false'
+        shared-key: engine
+        workspaces: src/rust/engine
+    - if: github.event_name != 'pull_request'
+      name: Setup toolchain auth
+      run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV
+
+        '
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.5
+    - env:
+        ARCHFLAGS: -arch arm64
+      name: Build wheels
+      run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex
+
+
+        USE_PY39=true ./build-support/bin/release.sh build-wheels'
+    - continue-on-error: true
+      if: always()
+      name: Upload pants.log
+      uses: actions/upload-artifact@v3
+      with:
+        name: pants-log-wheels-macOS11-ARM64
+        path: .pants.d/pants.log
+    - name
+    - run
+    - env
+    timeout-minutes: 90
+  determine_ref:
+    if: github.repository_owner == 'pantsbuild'
+    outputs:
+      build-ref: ${{ steps.determine_ref.outputs.build-ref }}
+      is-release: ${{ steps.determine_ref.outputs.is-release }}
     runs-on: ubuntu-latest
     steps:
     - env:
-        TAG: ${{ github.event.inputs.tag }}
-      id: determine-tag
-      name: Determine Release Tag
-      run: "if [[ -n \"$TAG\" ]]; then\n    tag=\"$TAG\"\nelse\n    tag=\"${GITHUB_REF#refs/tags/}\"\
-        \nfi\nif [[ \"${tag}\" =~ ^release_.+$ ]]; then\n    echo \"release-tag=${tag}\"\
-        \ >> $GITHUB_OUTPUT\nelse\n    echo \"::error::Release tag '${tag}' must match\
-        \ 'release_.+'.\"\n    exit 1\nfi\n"
+        REF: ${{ github.event.inputs.ref }}
+      id: determine_ref
+      name: Determine ref to build
+      run: "if [[ -n \"$REF\" ]]; then\n    ref=\"$REF\"\nelse\n    ref=\"${GITHUB_REF#refs/tags/}\"\
+        \nfi\necho \"build-ref=${ref}\" >> $GITHUB_OUTPUT\nif [[ \"${ref}\" =~ ^release_.+$\
+        \ ]]; then\n    echo \"is-release=true\" >> $GITHUB_OUTPUT\nfi\n"
+  publish:
+    if: github.repository_owner == 'pantsbuild' && needs.determine_ref.outputs.is-release
+      == 'true'
+    needs:
+    - build_wheels_linux_x86_64
+    - build_wheels_linux_arm64
+    - build_wheels_macos10_15_x86_64
+    - build_wheels_macos11_arm64
+    - determine_ref
+    runs-on: ubuntu-latest
+    steps:
     - name: Checkout Pants at Release Tag
       uses: actions/checkout@v3
       with:
-        ref: ${{ steps.determine-tag.outputs.release-tag }}
+        ref: ${{ needs.determine_ref.outputs.build-ref }}
+    - env:
+        MODE: debug
+      name: Fetch and stabilize wheels
+      run: ./build-support/bin/release.sh fetch-and-stabilize
     - name: Create Release -> Commit Mapping
-      run: 'tag="${{ steps.determine-tag.outputs.release-tag }}"
+      run: 'tag="${{ needs.determine_ref.outputs.build-ref }}"
 
         commit="$(git rev-parse ${tag}^{commit})"
 
@@ -33,19 +295,22 @@ jobs:
         echo "${commit}" > "dist/deploy/tags/pantsbuild.pants/${tag}"
 
         '
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PANTSBUILD_PYPI_API_TOKEN }}
     - env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-      name: Deploy to S3
+      name: Deploy commit mapping to S3
       run: ./build-support/bin/deploy_to_s3.py --scope tags/pantsbuild.pants
-name: Record Release Commit
+name: Release
 'on':
   push:
     tags:
     - release_*
   workflow_dispatch:
     inputs:
-      tag:
+      ref:
         required: true
         type: string
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -324,6 +324,7 @@ jobs:
     container:
       image: ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf
     env:
+      MODE: debug
       PANTS_REMOTE_CACHE_READ: 'false'
       PANTS_REMOTE_CACHE_WRITE: 'false'
     if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release
@@ -379,17 +380,12 @@ jobs:
       with:
         name: pants-log-wheels-Linux-ARM64
         path: .pants.d/pants.log
-    - env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: github.event_name == 'push'
-      name: Deploy to S3
-      run: ./build-support/bin/deploy_to_s3.py
     timeout-minutes: 90
   build_wheels_linux_x86_64:
     container:
       image: quay.io/pypa/manylinux2014_x86_64:latest
     env:
+      MODE: debug
       PANTS_REMOTE_CACHE_READ: 'false'
       PANTS_REMOTE_CACHE_WRITE: 'false'
     if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release
@@ -446,15 +442,10 @@ jobs:
       with:
         name: pants-log-wheels-Linux-x86_64
         path: .pants.d/pants.log
-    - env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: github.event_name == 'push'
-      name: Deploy to S3
-      run: ./build-support/bin/deploy_to_s3.py
     timeout-minutes: 90
   build_wheels_macos10_15_x86_64:
     env:
+      MODE: debug
       PANTS_REMOTE_CACHE_READ: 'false'
       PANTS_REMOTE_CACHE_WRITE: 'false'
     if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release
@@ -514,15 +505,10 @@ jobs:
       with:
         name: pants-log-wheels-macOS10-15-x86_64
         path: .pants.d/pants.log
-    - env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: github.event_name == 'push'
-      name: Deploy to S3
-      run: ./build-support/bin/deploy_to_s3.py
     timeout-minutes: 90
   build_wheels_macos11_arm64:
     env:
+      MODE: debug
       PANTS_REMOTE_CACHE_READ: 'false'
       PANTS_REMOTE_CACHE_WRITE: 'false'
     if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release
@@ -578,12 +564,6 @@ jobs:
       with:
         name: pants-log-wheels-macOS11-ARM64
         path: .pants.d/pants.log
-    - env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: github.event_name == 'push'
-      name: Deploy to S3
-      run: ./build-support/bin/deploy_to_s3.py
     timeout-minutes: 90
   check_labels:
     if: github.repository_owner == 'pantsbuild'