-
Notifications
You must be signed in to change notification settings - Fork 994
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: New workflow to build wheels (#2866)
* Add new build workflow Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Finally working version Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix Signed-off-by: Kevin Zhang <kzhang@tecton.ai>
- Loading branch information
Showing
2 changed files
with
278 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| name: build_wheels | ||
|
|
||
| on: workflow_dispatch | ||
|
|
||
| jobs: | ||
| get-version: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| release_version: ${{ steps.get_release_version.outputs.release_version }} | ||
| version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} | ||
| highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Get release version | ||
| id: get_release_version | ||
| run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} | ||
| - name: Get release version without prefix | ||
| id: get_release_version_without_prefix | ||
| env: | ||
| RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} | ||
| run: | | ||
| echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} | ||
| - name: Get highest semver | ||
| id: get_highest_semver | ||
| env: | ||
| RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} | ||
| run: | | ||
| source infra/scripts/setup-common-functions.sh | ||
| SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' | ||
| if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then | ||
| echo ::set-output name=highest_semver_tag::$(get_tag_release -m) | ||
| fi | ||
| - name: Check output | ||
| env: | ||
| RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} | ||
| VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} | ||
| HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} | ||
| run: | | ||
| echo $RELEASE_VERSION | ||
| echo $VERSION_WITHOUT_PREFIX | ||
| echo $HIGHEST_SEMVER_TAG | ||
| build-python-wheel: | ||
| name: Build wheels on ${{ matrix.os }} | ||
| runs-on: ${{ matrix.os }} | ||
| strategy: | ||
| matrix: | ||
| os: [ ubuntu-latest, macos-10.15 ] | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Setup Node | ||
| uses: actions/setup-node@v2 | ||
| with: | ||
| node-version: '17.x' | ||
| registry-url: 'https://registry.npmjs.org' | ||
| - name: Build UI | ||
| run: make build-ui | ||
| - name: Build wheels | ||
| uses: pypa/cibuildwheel@v2.7.0 | ||
| env: | ||
| CIBW_BUILD: "cp3*_x86_64" | ||
| CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" | ||
| CIBW_ARCHS: "native" | ||
| CIBW_ENVIRONMENT: > | ||
| COMPILE_GO=True PATH=$PATH:/usr/local/go/bin | ||
| CIBW_BEFORE_ALL_LINUX: | | ||
| curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz | ||
| tar -C /usr/local -xzf go.tar.gz | ||
| go version | ||
| CIBW_BEFORE_ALL_MACOS: | | ||
| curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg | ||
| sudo installer -pkg python.pkg -target / | ||
| CIBW_BEFORE_BUILD: | | ||
| make install-protoc-dependencies | ||
| make install-go-proto-dependencies | ||
| make install-go-ci-dependencies | ||
| - name: Build py310 specific wheels for macos | ||
| if: matrix.os == 'macos-10.15' | ||
| uses: pypa/cibuildwheel@v2.7.0 | ||
| env: | ||
| CIBW_BUILD: "cp310-macosx_x86_64" | ||
| CIBW_ARCHS: "native" | ||
| - uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: wheels | ||
| path: ./wheelhouse/*.whl | ||
|
|
||
| build-source-distribution: | ||
| name: Build source distribution | ||
| runs-on: macos-10.15 | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Setup Python | ||
| id: setup-python | ||
| uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: "3.10" | ||
| architecture: x64 | ||
| - name: Setup Node | ||
| uses: actions/setup-node@v2 | ||
| with: | ||
| node-version: '17.x' | ||
| registry-url: 'https://registry.npmjs.org' | ||
| - name: Build and install dependencies | ||
| run: | | ||
| pip install -U pip setuptools wheel twine | ||
| make install-protoc-dependencies | ||
| make install-go-proto-dependencies | ||
| make install-go-ci-dependencies | ||
| make build-ui | ||
| - name: Build | ||
| run: | | ||
| python3 setup.py sdist | ||
| - uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: wheels | ||
| path: dist/* | ||
|
|
||
| verify-python-wheels: | ||
| runs-on: ${{ matrix.os }} | ||
| needs: [build-python-wheel, build-source-distribution] | ||
| strategy: | ||
| matrix: | ||
| os: [ubuntu-latest, macos-10.15 ] | ||
| python-version: [ "3.7", "3.8", "3.9", "3.10"] | ||
| from-source: [ True, False ] | ||
| env: | ||
| # this script is for testing servers | ||
| # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself | ||
| TEST_SCRIPT: | | ||
| timeout 10s $@ & pid=$! | ||
| wait $pid | ||
| ret=$? | ||
| if [[ $ret -ne 124 ]] | ||
| then | ||
| exit $ret | ||
| else | ||
| echo "Succeeded!" | ||
| fi | ||
| steps: | ||
| - name: Setup Python | ||
| id: setup-python | ||
| uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| architecture: x64 | ||
| - uses: actions/setup-go@v3 | ||
| with: | ||
| go-version: '>=1.17.0' | ||
| - uses: actions/download-artifact@v2 | ||
| with: | ||
| name: wheels | ||
| path: dist | ||
| - name: Install wheel | ||
| if: ${{ !matrix.from-source }} | ||
| # try to install all wheels; only the current platform wheel should be actually installed | ||
| run: | | ||
| cd dist/ | ||
| pip install wheel | ||
| for f in *.whl; do pip install $f || true; done | ||
| - name: Install dist with go | ||
| if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}} | ||
| env: | ||
| COMPILE_GO: "True" | ||
| run: | | ||
| pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0' | ||
| go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 | ||
| go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 | ||
| pip install dist/*tar.gz | ||
| - name: Install dist w/o go | ||
| if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}} | ||
| run: pip install dist/*tar.gz | ||
| - name: Install OS X dependencies | ||
| if: matrix.os == 'macos-10.15' | ||
| run: brew install coreutils | ||
| - name: Smoke test | ||
| run: | | ||
| feast init test_repo | ||
| cd test_repo/ | ||
| feast apply | ||
| echo "$TEST_SCRIPT" > run-and-wait.sh | ||
| bash run-and-wait.sh feast serve | ||
| bash run-and-wait.sh feast ui | ||
| # We disable this test for the Python 3.10 binary since it does not include Go. | ||
| - name: Smoke test with go | ||
| if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest' | ||
| run: | | ||
| cd test_repo/ | ||
| feast apply | ||
| echo "$TEST_SCRIPT" > run-and-wait.sh | ||
| pip install cffi | ||
| printf "\ngo_feature_retrieval: True" >> feature_store.yaml | ||
| bash run-and-wait.sh feast serve |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters