From dc2f7faa127d938f0278455b4c3147199dfff443 Mon Sep 17 00:00:00 2001 From: atalman Date: Fri, 10 Jun 2022 13:55:31 -0700 Subject: [PATCH 1/3] Add m1 tagged build for torchtext --- .github/workflows/build-m1-binaries.yml | 28 ++++++++++++++++++++----- packaging/pkg_helpers.bash | 25 +++++++++++++++++----- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 27254c4ceb..762586a5c6 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -6,7 +6,13 @@ on: push: branches: - nightly + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +env: + CHANNEL: "nightly" jobs: build_wheels: name: "Build TorchText M1 wheels" @@ -17,6 +23,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Build TorchText M1 wheel shell: arch -arch arm64 bash {0} env: @@ -27,11 +40,16 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex . packaging/pkg_helpers.bash - setup_build_version + # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi git submodule update --init --recursive WHL_NAME=torchtext-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" @@ -46,19 +64,19 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchtext;print('torchtext version is ', torchtext.__version__)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} uses: actions/upload-artifact@v3 with: name: torchtext-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index cd92ab12e1..0d9848b15b 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -88,17 +88,32 @@ setup_cuda() { # Usage: setup_build_version setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - # version.txt for some reason has `a` character after major.minor.rev - # command below yields 0.10.0 from version.txt containing 0.10.0a0 - _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) - BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + if [[ -z "$1" ]]; then + setup_base_build_version + else + BUILD_VERSION="$1" + fi + BUILD_VERSION="$BUILD_VERSION.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi + + # Set build version based on tag if on tag + if [[ -n "${CIRCLE_TAG}" ]]; then + # Strip tag + BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}" + fi + export BUILD_VERSION } +setup_base_build_version() { + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + BUILD_VERSION=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + export BUILD_VERSION +} # Set some useful variables for OS X, if applicable setup_macos() { if [[ "$(uname)" == Darwin ]]; then From a0ddde17f7508e4472b92a6f9bd3c3365882df3b Mon Sep 17 00:00:00 2001 From: atalman Date: Fri, 10 Jun 2022 14:05:05 -0700 Subject: [PATCH 2/3] Remove default chanell --- .github/workflows/build-m1-binaries.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 762586a5c6..a13f6b8059 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -81,7 +81,6 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} - CHANNEL: nightly run: | for pkg in dist/*; do aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read From 996fa5e943435a2d4c38563477f37dd28b617ec0 Mon Sep 17 00:00:00 2001 From: atalman Date: Fri, 10 Jun 2022 14:10:25 -0700 Subject: [PATCH 3/3] prettier --- .github/workflows/build-m1-binaries.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index a13f6b8059..defc0a0dba 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -70,13 +70,17 @@ jobs: conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchtext;print('torchtext version is ', torchtext.__version__)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub - if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: + ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, + 'refs/tags/')) }} uses: actions/upload-artifact@v3 with: name: torchtext-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: + ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, + 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}