diff --git a/.github/workflows/m1.yaml b/.github/workflows/m1.yaml index 1f0da02b50c9..6eab562e7daf 100644 --- a/.github/workflows/m1.yaml +++ b/.github/workflows/m1.yaml @@ -1,27 +1,157 @@ -# Hand-edited for now, while we experiment with self-hosted M1 runners. -# -# TODO: Integrate into workflow yaml generator. -# -# NB: The runner is an X86_64 binary, and runs under Rosetta. -# Therefore, its subprocesses default to X86_64 as well. To force subprocesses, -# such wheel building, to run as ARM64, they must be invoked via `arch -arm64e`. -# This also means that ${{runner.arch }} reports "X64" rather than "ARM64", and so -# cannot be usefully used in jobs to disambiguate cache keys. - -name: M1 Wheel Build -'on': - push: - # TODO: Once we're satisfied with the job, change m1_test* to release_*, so we - # only build M1 wheels on release tags. - tags: - - m1test_* +# GENERATED, DO NOT EDIT! +# To change, edit `build-support/bin/generate_github_workflows.py` and run: +# ./pants run build-support/bin/generate_github_workflows.py + + +env: + ARCHFLAGS: -arch arm64e + PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} jobs: - dummy: + build_wheels_macos_arm64: + env: + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' if: ${{ github.repository_owner == 'pantsbuild' }} - name: "Dummy (macOS ARM64)" - runs-on: [self-hosted, macOS, ARM64] + name: Bootstrap Pants and build wheels (macOS-ARM64) + needs: check_labels + runs-on: + - self-hosted + - macOS + - ARM64 steps: - - name: Echo something - run: | - arch -arm64e echo "Job was triggered on a runner that thinks it is "\ - "${{ runner.os }} ${{ runner.arch }}, but can actually be $(arch -arm64e uname -p)." + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - if: github.event_name == 'push' + name: Get commit message for branch builds + run: 'echo "COMMIT_MESSAGE<> $GITHUB_ENV + + echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV + + echo "EOF" >> $GITHUB_ENV + + ' + - if: github.event_name == 'pull_request' + name: Get commit message for PR builds + run: 'echo "COMMIT_MESSAGE<> $GITHUB_ENV + + echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV + + echo "EOF" >> $GITHUB_ENV + + ' + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Cache Rust toolchain + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}-v1 + path: '~/.rustup/toolchains/1.61.0-* + + ~/.rustup/update-hashes + + ~/.rustup/settings.toml + + ' + - name: Cache Cargo + uses: actions/cache@v3 + with: + key: '${{ runner.os }}-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') + }}-v1 + + ' + path: '~/.cargo/registry + + ~/.cargo/git + + ' + restore-keys: '${{ runner.os }}-cargo-${{ hashFiles(''rust-toolchain'') }}- + + ' + - id: get-engine-hash + name: Get native engine hash + run: 'echo "::set-output name=hash::$(./build-support/bin/rust/print_engine_hash.sh)" + + ' + shell: bash + - name: Cache native engine + uses: actions/cache@v3 + with: + key: '${{ runner.os }}-engine-${{ steps.get-engine-hash.outputs.hash }}-v1 + + ' + path: '.pants + + src/python/pants/engine/internals/native_engine.so + + src/python/pants/engine/internals/native_engine.so.metadata' + - if: github.event_name != 'pull_request' + name: Setup toolchain auth + run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV + + ' + - name: Bootstrap Pants + run: 'arch -arm64e ./pants --version + + ' + - name: Run smoke tests + run: 'arch -arm64e ./pants list :: + + arch -arm64e ./pants roots + + arch -arm64e ./pants help goals + + arch -arm64e ./pants help targets + + arch -arm64e ./pants help subsystems + + ' + - if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-bootstrap-macOS-ARM64 + path: .pants.d/pants.log + - name: Upload native binaries + uses: actions/upload-artifact@v3 + with: + name: native_binaries.${ matrix.python-version }.macOS-ARM64 + path: '.pants + + src/python/pants/engine/internals/native_engine.so + + src/python/pants/engine/internals/native_engine.so.metadata' + - if: (github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]')) + && (${{ github.repository_owner == 'pantsbuild' }}) + name: Build wheels + run: '[[ "${GITHUB_EVENT_NAME}" == "pull_request" ]] && export MODE=debug + + # TODO: Build local pex? Will require some changes to _release_helper.py to + qualify + + # the .pex file name with the architecture, intead of just "darwin". + + #USE_PY39=true arch -arm64e ./build-support/bin/release.sh build-local-pex + + USE_PY39=true arch -arm64e ./build-support/bin/release.sh build-wheels + + ' + strategy: + matrix: + python-version: + - '3.9' + timeout-minutes: 60 +name: Build wheels (macOS arm64) +'on': + pull_request: {} + push: + branches-ignore: + - dependabot/** diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml index b1c02b25d74f..aca64eb70392 100644 --- a/.github/workflows/test-cron.yaml +++ b/.github/workflows/test-cron.yaml @@ -6,15 +6,17 @@ env: PANTS_CONFIG_FILES: +['pants.ci.toml'] RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} jobs: bootstrap_pants_linux: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ${{ github.repository_owner == 'pantsbuild' }} - name: Bootstrap Pants, test+lint Rust (Linux) + name: Bootstrap Pants, test and lint Rust (Linux-x86_64) needs: check_labels - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -100,10 +102,6 @@ jobs: - name: Bootstrap Pants run: './pants --version - ' - - name: Validate CI config - run: './pants run build-support/bin/generate_github_workflows.py -- --check - ' - name: Run smoke tests run: './pants list :: @@ -121,19 +119,23 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-bootstrap-linux + name: pants-log-bootstrap-Linux-x86_64 path: .pants.d/pants.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 path: '.pants src/python/pants/engine/internals/native_engine.so src/python/pants/engine/internals/native_engine.so.metadata' + - name: Validate CI config + run: './pants run build-support/bin/generate_github_workflows.py -- --check + + ' - if: '!contains(env.COMMIT_MESSAGE, ''[ci skip-rust]'')' - name: Test and Lint Rust + name: Test and lint Rust run: 'sudo apt-get install -y pkg-config fuse libfuse-dev ./build-support/bin/check_rust_pre_commit.sh @@ -154,20 +156,11 @@ jobs: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ${{ github.repository_owner == 'pantsbuild' }} - name: Bootstrap Pants, test Rust (macOS) + name: Bootstrap Pants, test Rust (macOS-x86_64) needs: check_labels - runs-on: macos-10.15 + runs-on: + - macos-10.15 steps: - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: github.event_name == 'pull_request' - name: Ensure category label - uses: mheap/github-action-required-labels@v1 - with: - count: 1 - labels: category:new feature, category:user api change, category:plugin api - change, category:performance, category:bugfix, category:documentation, category:internal - mode: exactly - name: Check out code uses: actions/checkout@v3 with: @@ -253,18 +246,34 @@ jobs: run: './pants --version ' + - name: Run smoke tests + run: './pants list :: + + ./pants roots + + ./pants help goals + + ./pants help targets + + ./pants help subsystems + + ' + - if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-bootstrap-macOS-x86_64 + path: .pants.d/pants.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.macOS-x86_64 path: '.pants src/python/pants/engine/internals/native_engine.so src/python/pants/engine/internals/native_engine.so.metadata' - - env: - TMPDIR: ${{ runner.temp }} - if: '!contains(env.COMMIT_MESSAGE, ''[ci skip-rust]'')' + - if: '!contains(env.COMMIT_MESSAGE, ''[ci skip-rust]'')' name: Test Rust run: ./cargo test --tests -- --nocapture strategy: @@ -276,7 +285,8 @@ jobs: check_labels: if: ${{ github.repository_owner == 'pantsbuild' }} name: Ensure PR has a category label - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -292,7 +302,8 @@ jobs: if: ${{ github.repository_owner == 'pantsbuild' }} name: Lint Python and Shell needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -330,7 +341,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -356,9 +367,10 @@ jobs: timeout-minutes: 30 test_python_linux_0: if: ${{ github.repository_owner == 'pantsbuild' }} - name: Test Python (Linux) Shard 0/3 + name: Test Python (Linux-x86_64) Shard 0/3 needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -419,7 +431,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -443,9 +455,10 @@ jobs: timeout-minutes: 90 test_python_linux_1: if: ${{ github.repository_owner == 'pantsbuild' }} - name: Test Python (Linux) Shard 1/3 + name: Test Python (Linux-x86_64) Shard 1/3 needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -506,7 +519,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -530,9 +543,10 @@ jobs: timeout-minutes: 90 test_python_linux_2: if: ${{ github.repository_owner == 'pantsbuild' }} - name: Test Python (Linux) Shard 2/3 + name: Test Python (Linux-x86_64) Shard 2/3 needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -593,7 +607,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -618,10 +632,14 @@ jobs: test_python_macos: env: ARCHFLAGS: -arch x86_64 + PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} if: ${{ github.repository_owner == 'pantsbuild' }} name: Test Python (macOS) needs: bootstrap_pants_macos - runs-on: macos-10.15 + runs-on: + - macos-10.15 steps: - name: Check out code uses: actions/checkout@v3 @@ -666,7 +684,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.macOS-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 66806d048c45..ed23b666fb19 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,15 +6,17 @@ env: PANTS_CONFIG_FILES: +['pants.ci.toml'] RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} jobs: bootstrap_pants_linux: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ${{ github.repository_owner == 'pantsbuild' }} - name: Bootstrap Pants, test+lint Rust (Linux) + name: Bootstrap Pants, test and lint Rust (Linux-x86_64) needs: check_labels - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -100,10 +102,6 @@ jobs: - name: Bootstrap Pants run: './pants --version - ' - - name: Validate CI config - run: './pants run build-support/bin/generate_github_workflows.py -- --check - ' - name: Run smoke tests run: './pants list :: @@ -121,19 +119,23 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-bootstrap-linux + name: pants-log-bootstrap-Linux-x86_64 path: .pants.d/pants.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 path: '.pants src/python/pants/engine/internals/native_engine.so src/python/pants/engine/internals/native_engine.so.metadata' + - name: Validate CI config + run: './pants run build-support/bin/generate_github_workflows.py -- --check + + ' - if: '!contains(env.COMMIT_MESSAGE, ''[ci skip-rust]'')' - name: Test and Lint Rust + name: Test and lint Rust run: 'sudo apt-get install -y pkg-config fuse libfuse-dev ./build-support/bin/check_rust_pre_commit.sh @@ -153,20 +155,11 @@ jobs: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ${{ github.repository_owner == 'pantsbuild' }} - name: Bootstrap Pants, test Rust (macOS) + name: Bootstrap Pants, test Rust (macOS-x86_64) needs: check_labels - runs-on: macos-10.15 + runs-on: + - macos-10.15 steps: - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: github.event_name == 'pull_request' - name: Ensure category label - uses: mheap/github-action-required-labels@v1 - with: - count: 1 - labels: category:new feature, category:user api change, category:plugin api - change, category:performance, category:bugfix, category:documentation, category:internal - mode: exactly - name: Check out code uses: actions/checkout@v3 with: @@ -252,18 +245,34 @@ jobs: run: './pants --version ' + - name: Run smoke tests + run: './pants list :: + + ./pants roots + + ./pants help goals + + ./pants help targets + + ./pants help subsystems + + ' + - if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-bootstrap-macOS-x86_64 + path: .pants.d/pants.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.macOS-x86_64 path: '.pants src/python/pants/engine/internals/native_engine.so src/python/pants/engine/internals/native_engine.so.metadata' - - env: - TMPDIR: ${{ runner.temp }} - if: '!contains(env.COMMIT_MESSAGE, ''[ci skip-rust]'')' + - if: '!contains(env.COMMIT_MESSAGE, ''[ci skip-rust]'')' name: Test Rust run: ./cargo test --tests -- --nocapture strategy: @@ -322,6 +331,8 @@ jobs: ' - env: PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]') name: Build wheels run: '[[ "${GITHUB_EVENT_NAME}" == "pull_request" ]] && export MODE=debug @@ -339,6 +350,8 @@ jobs: ' - env: PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} if: github.event_name == 'push' name: Build fs_util run: ./build-support/bin/release.sh build-fs-util @@ -346,7 +359,7 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-linux + name: pants-log-wheels-linux-x86_64 path: .pants.d/pants.log - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -422,6 +435,8 @@ jobs: - env: ARCHFLAGS: -arch x86_64 PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]') name: Build wheels run: '[[ "${GITHUB_EVENT_NAME}" == "pull_request" ]] && export MODE=debug @@ -440,6 +455,8 @@ jobs: - env: ARCHFLAGS: -arch x86_64 PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} if: github.event_name == 'push' name: Build fs_util run: ./build-support/bin/release.sh build-fs-util @@ -447,7 +464,7 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-macos + name: pants-log-wheels-macos-x86_64 path: .pants.d/pants.log - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -459,7 +476,8 @@ jobs: check_labels: if: ${{ github.repository_owner == 'pantsbuild' }} name: Ensure PR has a category label - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -475,7 +493,8 @@ jobs: if: ${{ github.repository_owner == 'pantsbuild' }} name: Lint Python and Shell needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -513,7 +532,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -538,9 +557,10 @@ jobs: timeout-minutes: 30 test_python_linux_0: if: ${{ github.repository_owner == 'pantsbuild' }} - name: Test Python (Linux) Shard 0/3 + name: Test Python (Linux-x86_64) Shard 0/3 needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -601,7 +621,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -624,9 +644,10 @@ jobs: timeout-minutes: 90 test_python_linux_1: if: ${{ github.repository_owner == 'pantsbuild' }} - name: Test Python (Linux) Shard 1/3 + name: Test Python (Linux-x86_64) Shard 1/3 needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -687,7 +708,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -710,9 +731,10 @@ jobs: timeout-minutes: 90 test_python_linux_2: if: ${{ github.repository_owner == 'pantsbuild' }} - name: Test Python (Linux) Shard 2/3 + name: Test Python (Linux-x86_64) Shard 2/3 needs: bootstrap_pants_linux - runs-on: ubuntu-20.04 + runs-on: + - ubuntu-20.04 steps: - name: Check out code uses: actions/checkout@v3 @@ -773,7 +795,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.Linux-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV @@ -797,10 +819,14 @@ jobs: test_python_macos: env: ARCHFLAGS: -arch x86_64 + PANTS_CONFIG_FILES: +['pants.ci.toml'] + RUST_BACKTRACE: all + TMPDIR: ${{ runner.temp }} if: ${{ github.repository_owner == 'pantsbuild' }} name: Test Python (macOS) needs: bootstrap_pants_macos - runs-on: macos-10.15 + runs-on: + - macos-10.15 steps: - name: Check out code uses: actions/checkout@v3 @@ -845,7 +871,7 @@ jobs: - name: Download native binaries uses: actions/download-artifact@v3 with: - name: native_binaries.${{ matrix.python-version }}.${{ runner.os }} + name: native_binaries.${ matrix.python-version }.macOS-x86_64 - if: github.event_name != 'pull_request' name: Setup toolchain auth run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV diff --git a/build-support/bin/_release_helper.py b/build-support/bin/_release_helper.py index cb5cfa4584d6..951da99b797f 100644 --- a/build-support/bin/_release_helper.py +++ b/build-support/bin/_release_helper.py @@ -783,6 +783,7 @@ def build_fs_util() -> None: def build_pex(fetch: bool) -> None: stable = os.environ.get("PANTS_PEX_RELEASE", "") == "STABLE" if fetch: + # TODO: Support macOS on ARM64. extra_pex_args = [ "--python-shebang", "/usr/bin/env python", @@ -797,6 +798,7 @@ def build_pex(fetch: bool) -> None: pex_name = f"pants.{CONSTANTS.pants_unstable_version}.pex" banner(f"Building {pex_name} by fetching wheels.") else: + # TODO: Support macOS on ARM64. Will require qualifying the pex name with the arch. major, minor = sys.version_info[:2] extra_pex_args = [ f"--interpreter-constraint=CPython=={major}.{minor}.*", diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index bbb047233f2c..61cb6c24ec0b 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -6,6 +6,7 @@ import argparse import os from dataclasses import dataclass +from enum import Enum from pathlib import Path from textwrap import dedent from typing import Any, Dict, Sequence, cast @@ -30,6 +31,12 @@ Env = Dict[str, str] +class Platform(Enum): + LINUX_X86_64 = "Linux-x86_64" + MACOS_X86_64 = "macOS-x86_64" + MACOS_ARM64 = "macOS-ARM64" + + # ---------------------------------------------------------------------- # Constants # ---------------------------------------------------------------------- @@ -58,9 +65,6 @@ # NB: This overrides `pants.ci.toml`. DISABLE_REMOTE_CACHE_ENV = {"PANTS_REMOTE_CACHE_READ": "false", "PANTS_REMOTE_CACHE_WRITE": "false"} -# Works around bad `-arch arm64` flag embedded in Xcode 12.x Python interpreters on -# intel machines. See: https://github.com/giampaolo/psutil/issues/1832 -MACOS_ENV = {"ARCHFLAGS": "-arch x86_64"} IS_PANTS_OWNER = "${{ github.repository_owner == 'pantsbuild' }}" @@ -165,6 +169,7 @@ def global_env() -> Env: return { "PANTS_CONFIG_FILES": "+['pants.ci.toml']", "RUST_BACKTRACE": "all", + "TMPDIR": "${{ runner.temp }}", } @@ -227,6 +232,18 @@ def install_go() -> Step: } +def deploy_to_s3() -> Step: + return { + "name": "Deploy to S3", + "run": "./build-support/bin/deploy_to_s3.py", + "if": "github.event_name == 'push'", + "env": { + "AWS_SECRET_ACCESS_KEY": "${{ secrets.AWS_SECRET_ACCESS_KEY }}", + "AWS_ACCESS_KEY_ID": "${{ secrets.AWS_ACCESS_KEY_ID }}", + }, + } + + def bootstrap_caches() -> Sequence[Step]: return [ *rust_caches(), @@ -251,32 +268,17 @@ def bootstrap_caches() -> Sequence[Step]: ] -def native_binaries_upload() -> Step: - return { - "name": "Upload native binaries", - "uses": "actions/upload-artifact@v3", - "with": { - "name": "native_binaries.${{ matrix.python-version }}.${{ runner.os }}", - "path": "\n".join(NATIVE_FILES), - }, - } - - -def native_binaries_download() -> Step: - return { - "name": "Download native binaries", - "uses": "actions/download-artifact@v3", - "with": {"name": "native_binaries.${{ matrix.python-version }}.${{ runner.os }}"}, - } - - -def setup_primary_python() -> Sequence[Step]: - return [ - { - "name": "Set up Python ${{ matrix.python-version }}", - "uses": "actions/setup-python@v3", - "with": {"python-version": "${{ matrix.python-version }}"}, - }, +def setup_primary_python(install_python: bool = True) -> Sequence[Step]: + ret = [] + if install_python: + ret.append( + { + "name": "Set up Python ${{ matrix.python-version }}", + "uses": "actions/setup-python@v3", + "with": {"python-version": "${{ matrix.python-version }}"}, + } + ) + ret.append( { "name": "Tell Pants to use Python ${{ matrix.python-version }}", "run": dedent( @@ -285,8 +287,9 @@ def setup_primary_python() -> Sequence[Step]: echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=['==${{ matrix.python-version }}.*']" >> $GITHUB_ENV """ ), - }, - ] + } + ) + return ret def expose_all_pythons() -> Step: @@ -320,11 +323,111 @@ def download_apache_thrift() -> Step: } +class Helper: + def __init__(self, platform: Platform): + self.platform = platform + + def platform_name(self) -> str: + return str(self.platform.value) + + def runs_on(self) -> list[str]: + if self.platform == Platform.MACOS_X86_64: + return [MACOS_VERSION] + if self.platform == Platform.MACOS_ARM64: + return ["self-hosted", "macOS", "ARM64"] + if self.platform == Platform.LINUX_X86_64: + return [LINUX_VERSION] + raise ValueError(f"Unsupported platform: {self.platform_name()}") + + def env(self): + ret = dict(global_env()) + if self.platform == Platform.MACOS_X86_64: + # Works around bad `-arch arm64` flag embedded in Xcode 12.x Python interpreters on + # intel machines. See: https://github.com/giampaolo/psutil/issues/1832 + ret["ARCHFLAGS"] = "-arch x86_64" + if self.platform == Platform.MACOS_ARM64: + ret["ARCHFLAGS"] = "-arch arm64e" + return ret + + def wrap_cmd(self, cmd: str) -> str: + if self.platform == Platform.MACOS_ARM64: + # The self-hosted M1 runner is an X86_64 binary that runs under Rosetta, + # so we have to explicitly change the arch for the subprocesses it spawns. + return f"arch -arm64e {cmd}" + return cmd + + def native_binaries_upload(self) -> Step: + return { + "name": "Upload native binaries", + "uses": "actions/upload-artifact@v3", + "with": { + "name": f"native_binaries.${{ matrix.python-version }}.{self.platform_name()}", + "path": "\n".join(NATIVE_FILES), + }, + } + + def native_binaries_download(self) -> Step: + return { + "name": "Download native binaries", + "uses": "actions/download-artifact@v3", + "with": { + "name": f"native_binaries.${{ matrix.python-version }}.{self.platform_name()}", + }, + } + + def rust_caches(self) -> Sequence[Step]: + return [ + { + "name": "Cache Rust toolchain", + "uses": "actions/cache@v3", + "with": { + "path": f"~/.rustup/toolchains/{rust_channel()}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n", + "key": f"{self.platform_name()}-rustup-${{ hashFiles('rust-toolchain') }}-v1", + }, + }, + { + "name": "Cache Cargo", + "uses": "actions/cache@v3", + "with": { + "path": "~/.cargo/registry\n~/.cargo/git\n", + "key": f"{self.platform_name()}-cargo-${{ hashFiles('rust-toolchain') }}-${{ hashFiles('src/rust/engine/Cargo.*') }}-v1\n", + "restore-keys": f"{self.platform_name()}-cargo-${{ hashFiles('rust-toolchain') }}-\n", + }, + }, + ] + + def bootstrap_pants(self, *, install_python: bool) -> Sequence[Step]: + return [ + *checkout(), + *setup_primary_python(install_python=install_python), + *bootstrap_caches(), + setup_toolchain_auth(), + {"name": "Bootstrap Pants", "run": self.wrap_cmd("./pants --version\n")}, + { + "name": "Run smoke tests", + "run": dedent( + f"""\ + {self.wrap_cmd("./pants list ::")} + {self.wrap_cmd("./pants roots")} + {self.wrap_cmd("./pants help goals")} + {self.wrap_cmd("./pants help targets")} + {self.wrap_cmd("./pants help subsystems")} + """ + ), + }, + upload_log_artifacts(name=f"bootstrap-{self.platform_name()}"), + self.native_binaries_upload(), + ] + + def test_workflow_jobs(python_versions: list[str], *, cron: bool) -> Jobs: + helper_linux = Helper(Platform.LINUX_X86_64) + helper_macos = Helper(Platform.MACOS_X86_64) + def test_python_linux(shard: str) -> dict[str, Any]: return { - "name": f"Test Python (Linux) Shard {shard}", - "runs-on": LINUX_VERSION, + "name": f"Test Python ({helper_linux.platform_name()}) Shard {shard}", + "runs-on": helper_linux.runs_on(), "needs": "bootstrap_pants_linux", "strategy": {"matrix": {"python-version": python_versions}}, "timeout-minutes": 90, @@ -336,7 +439,7 @@ def test_python_linux(shard: str) -> dict[str, Any]: download_apache_thrift(), *setup_primary_python(), expose_all_pythons(), - native_binaries_download(), + helper_linux.native_binaries_download(), setup_toolchain_auth(), { "name": f"Run Python test shard {shard}", @@ -349,24 +452,20 @@ def test_python_linux(shard: str) -> dict[str, Any]: jobs = { "check_labels": { "name": "Ensure PR has a category label", - "runs-on": LINUX_VERSION, + "runs-on": helper_linux.runs_on(), "if": IS_PANTS_OWNER, "steps": ensure_category_label(), }, "bootstrap_pants_linux": { - "name": "Bootstrap Pants, test+lint Rust (Linux)", - "runs-on": LINUX_VERSION, + "name": f"Bootstrap Pants, test and lint Rust ({helper_linux.platform_name()})", + "runs-on": helper_linux.runs_on(), "needs": "check_labels", "strategy": {"matrix": {"python-version": python_versions}}, "env": DISABLE_REMOTE_CACHE_ENV, "timeout-minutes": 40, "if": IS_PANTS_OWNER, "steps": [ - *checkout(), - *setup_primary_python(), - *bootstrap_caches(), - setup_toolchain_auth(), - {"name": "Bootstrap Pants", "run": "./pants --version\n"}, + *helper_linux.bootstrap_pants(install_python=True), { "name": "Validate CI config", "run": dedent( @@ -376,21 +475,7 @@ def test_python_linux(shard: str) -> dict[str, Any]: ), }, { - "name": "Run smoke tests", - "run": dedent( - """\ - ./pants list :: - ./pants roots - ./pants help goals - ./pants help targets - ./pants help subsystems - """ - ), - }, - upload_log_artifacts(name="bootstrap-linux"), - native_binaries_upload(), - { - "name": "Test and Lint Rust", + "name": "Test and lint Rust", # We pass --tests to skip doc tests because our generated protos contain # invalid doc tests in their comments. "run": dedent( @@ -410,7 +495,7 @@ def test_python_linux(shard: str) -> dict[str, Any]: "test_python_linux_2": test_python_linux("2/3"), "lint_python": { "name": "Lint Python and Shell", - "runs-on": LINUX_VERSION, + "runs-on": helper_linux.runs_on(), "needs": "bootstrap_pants_linux", "strategy": {"matrix": {"python-version": python_versions}}, "timeout-minutes": 30, @@ -418,7 +503,7 @@ def test_python_linux(shard: str) -> dict[str, Any]: "steps": [ *checkout(), *setup_primary_python(), - native_binaries_download(), + helper_linux.native_binaries_download(), setup_toolchain_auth(), { "name": "Lint", @@ -432,38 +517,31 @@ def test_python_linux(shard: str) -> dict[str, Any]: ], }, "bootstrap_pants_macos": { - "name": "Bootstrap Pants, test Rust (macOS)", - "runs-on": MACOS_VERSION, + "name": f"Bootstrap Pants, test Rust ({helper_macos.platform_name()})", + "runs-on": helper_macos.runs_on(), "needs": "check_labels", "strategy": {"matrix": {"python-version": python_versions}}, "env": DISABLE_REMOTE_CACHE_ENV, "timeout-minutes": 40, "if": IS_PANTS_OWNER, "steps": [ - *ensure_category_label(), - *checkout(), - *setup_primary_python(), - *bootstrap_caches(), - setup_toolchain_auth(), - {"name": "Bootstrap Pants", "run": "./pants --version\n"}, - native_binaries_upload(), + *helper_macos.bootstrap_pants(install_python=True), { "name": "Test Rust", # We pass --tests to skip doc tests because our generated protos contain # invalid doc tests in their comments. We do not pass --all as BRFS tests don't # pass on GHA MacOS containers. - "run": "./cargo test --tests -- --nocapture", + "run": helper_macos.wrap_cmd("./cargo test --tests -- --nocapture"), "if": DONT_SKIP_RUST, - "env": {"TMPDIR": "${{ runner.temp }}"}, }, ], }, "test_python_macos": { "name": "Test Python (macOS)", - "runs-on": MACOS_VERSION, + "runs-on": helper_macos.runs_on(), "needs": "bootstrap_pants_macos", "strategy": {"matrix": {"python-version": python_versions}}, - "env": MACOS_ENV, + "env": helper_macos.env(), "timeout-minutes": 60, "if": IS_PANTS_OWNER, "steps": [ @@ -471,7 +549,7 @@ def test_python_linux(shard: str) -> dict[str, Any]: install_jdk(), *setup_primary_python(), expose_all_pythons(), - native_binaries_download(), + helper_macos.native_binaries_download(), setup_toolchain_auth(), { "name": "Run Python tests", @@ -488,8 +566,8 @@ def test_python_linux(shard: str) -> dict[str, Any]: } if not cron: - def build_steps(*, is_macos: bool) -> list[Step]: - env = {"PANTS_CONFIG_FILES": "+['pants.ci.toml']", **(MACOS_ENV if is_macos else {})} + def build_steps(*, platform: Platform) -> list[Step]: + helper = Helper(platform) return [ { "name": "Build wheels", @@ -510,7 +588,7 @@ def build_steps(*, is_macos: bool) -> list[Step]: """ ), "if": DONT_SKIP_WHEELS, - "env": env, + "env": helper.env(), }, { "name": "Build fs_util", @@ -518,7 +596,7 @@ def build_steps(*, is_macos: bool) -> list[Step]: # We only build fs_util on branch builds, given that Pants compilation already # checks the code compiles and the release process is simple and low-stakes. "if": "github.event_name == 'push'", - "env": env, + "env": helper.env(), }, ] @@ -527,15 +605,6 @@ def build_steps(*, is_macos: bool) -> list[Step]: "env": DISABLE_REMOTE_CACHE_ENV, "if": IS_PANTS_OWNER, } - deploy_to_s3_step = { - "name": "Deploy to S3", - "run": "./build-support/bin/deploy_to_s3.py", - "if": "github.event_name == 'push'", - "env": { - "AWS_SECRET_ACCESS_KEY": "${{ secrets.AWS_SECRET_ACCESS_KEY }}", - "AWS_ACCESS_KEY_ID": "${{ secrets.AWS_ACCESS_KEY_ID }}", - }, - } jobs.update( { "build_wheels_linux_x86_64": { @@ -557,9 +626,9 @@ def build_steps(*, is_macos: bool) -> list[Step]: ), }, setup_toolchain_auth(), - *build_steps(is_macos=False), - upload_log_artifacts(name="wheels-linux"), - deploy_to_s3_step, + *build_steps(platform=Platform.LINUX_X86_64), + upload_log_artifacts(name="wheels-linux-x86_64"), + deploy_to_s3(), ], }, "build_wheels_macos_x86_64": { @@ -572,13 +641,13 @@ def build_steps(*, is_macos: bool) -> list[Step]: setup_toolchain_auth(), expose_all_pythons(), # NB: We only cache Rust, but not `native_engine.so` and the Pants - # virtualenv. This is because we must build both these things with Python + # virtualenv. This is because we must build both these things with # multiple Python versions, whereas that caching assumes only one primary # Python version (marked via matrix.strategy). *rust_caches(), - *build_steps(is_macos=True), - upload_log_artifacts(name="wheels-macos"), - deploy_to_s3_step, + *build_steps(platform=Platform.MACOS_X86_64), + upload_log_artifacts(name="wheels-macos-x86_64"), + deploy_to_s3(), ], }, } @@ -719,6 +788,7 @@ def generate() -> dict[Path, str]: }, Dumper=NoAliasDumper, ) + test_cron_yaml = yaml.dump( { "name": "Daily Extended Python Testing", @@ -730,6 +800,61 @@ def generate() -> dict[Path, str]: Dumper=NoAliasDumper, ) + helper_macos_arm64 = Helper(Platform.MACOS_ARM64) + steps = list(helper_macos_arm64.bootstrap_pants(install_python=False)) + steps.append( + { + "name": "Build wheels", + "run": dedent( + # We use MODE=debug on PR builds to speed things up, given that those are + # only smoke tests of our release process. + # Note that the build-local-pex run is just for smoke-testing that pex + # builds work, and it must come *before* the build-wheels runs, since + # it cleans out `dist/deploy`, which the build-wheels runs populate for + # later attention by deploy_to_s3.py. + f"""\ + [[ "${{GITHUB_EVENT_NAME}}" == "pull_request" ]] && export MODE=debug + # TODO: Build local pex? Will require some changes to _release_helper.py to qualify + # the .pex file name with the architecture, intead of just "darwin". + #USE_PY39=true {helper_macos_arm64.wrap_cmd("./build-support/bin/release.sh build-local-pex")} + USE_PY39=true {helper_macos_arm64.wrap_cmd("./build-support/bin/release.sh build-wheels")} + """ + ), + "if": f"({DONT_SKIP_WHEELS}) && ({IS_PANTS_OWNER})", + } + ) + job = { + "name": f"Bootstrap Pants and build wheels ({Platform.MACOS_ARM64.value})", + "runs-on": helper_macos_arm64.runs_on(), + "needs": "check_labels", + "strategy": {"matrix": {"python-version": [PYTHON39_VERSION]}}, + "timeout-minutes": 60, + "if": IS_PANTS_OWNER, + "steps": steps, + "env": DISABLE_REMOTE_CACHE_ENV, + } + + macos_arm64_yaml = yaml.dump( + { + "name": "Build wheels (macOS arm64)", + "on": { + # TODO: If there is too much contention for resources on our lone M1 self-hosted + # runner, remove the pull_request line, and only build wheels on pushes to main. + "pull_request": {}, + "push": { + "branches-ignore": ["dependabot/**"], + }, + }, + # The setup-python action doesn't yet support installing ARM64 Pythons. + # Instead we pre-install Python 3.9 on the self-hosted runner. + "jobs": { + "build_wheels_macos_arm64": job, + }, + "env": helper_macos_arm64.env(), + }, + Dumper=NoAliasDumper, + ) + cancel_yaml = yaml.dump( { # Note that this job runs in the context of the default branch, so its token @@ -799,6 +924,7 @@ def generate() -> dict[Path, str]: Path(".github/workflows/cancel.yaml"): f"{HEADER}\n\n{cancel_yaml}", Path(".github/workflows/test.yaml"): f"{HEADER}\n\n{test_yaml}", Path(".github/workflows/test-cron.yaml"): f"{HEADER}\n\n{test_cron_yaml}", + Path(".github/workflows/m1.yaml"): f"{HEADER}\n\n{macos_arm64_yaml}", }