ci: New workflow to build wheels

.github/workflows/build_wheels.yml

@@ -0,0 +1,193 @@
+name: build_wheels
+
+on: workflow_dispatch
+
+jobs:
+  get-version:
+    runs-on:  ubuntu-latest
+    outputs:
+      release_version: ${{ steps.get_release_version.outputs.release_version }}
+      version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
+      highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get release version
+        id: get_release_version
+        run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
+      - name: Get release version without prefix
+        id: get_release_version_without_prefix
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+        run: |
+          echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
+      - name: Get highest semver
+        id: get_highest_semver
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+        run: |
+          source infra/scripts/setup-common-functions.sh
+          SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
+          if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
+            echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
+          fi
+      - name: Check output
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
+          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
+        run: |
+          echo $RELEASE_VERSION
+          echo $VERSION_WITHOUT_PREFIX
+          echo $HIGHEST_SEMVER_TAG
+
+  build-python-wheel:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-10.15 ]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '17.x'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Build UI
+        run: make build-ui
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.7.0
+        env:
+          CIBW_BUILD: "cp3*_x86_64"
+          CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
+          CIBW_ARCHS: "native"
+          CIBW_ENVIRONMENT: >
+            COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
+          CIBW_BEFORE_ALL_LINUX: |
+            curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
+            tar -C /usr/local -xzf go.tar.gz
+            go version
+          CIBW_BEFORE_ALL_MACOS: |
+            curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
+            sudo installer -pkg python.pkg -target /
+          CIBW_BEFORE_BUILD: |
+            make install-protoc-dependencies
+            make install-go-proto-dependencies
+            make install-go-ci-dependencies
+      - name: Build py310 specific wheels for macos
+        if: matrix.os == 'macos-10.15'
+        uses: pypa/cibuildwheel@v2.7.0
+        env:
+          CIBW_BUILD: "cp310-macosx_x86_64"
+          CIBW_ARCHS: "native"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
+
+  build-source-distribution:
+    name: Build source distribution
+    runs-on: macos-10.15
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+          architecture: x64
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '17.x'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Build and install dependencies
+        run: |
+          pip install -U pip setuptools wheel twine
+          make install-protoc-dependencies
+          make install-go-proto-dependencies
+          make install-go-ci-dependencies
+          make build-ui
+      - name: Build
+        run: |
+          python3 setup.py sdist
+      - uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: dist/*
+
+  verify-python-wheels:
+    runs-on: ${{ matrix.os }}
+    needs: [build-python-wheel, build-source-distribution]
+    strategy:
+      matrix:
+        os: [ubuntu-latest,  macos-10.15 ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10"]
+        from-source: [ True, False ]
+    env:
+      # this script is for testing servers
+      # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
+      TEST_SCRIPT: |
+        timeout 10s $@ & pid=$!
+        wait $pid
+        ret=$?
+        if [[ $ret -ne 124 ]]
+        then
+           exit $ret
+        else
+           echo "Succeeded!"
+        fi
+    steps:
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.17.0'
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+          path: dist
+      - name: Install wheel
+        if: ${{ !matrix.from-source }}
+        # try to install all wheels; only the current platform wheel should be actually installed
+        run: |
+          cd dist/
+          pip install wheel
+          for f in *.whl; do pip install $f || true; done
+      - name: Install dist with go
+        if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}}
+        env:
+          COMPILE_GO: "True"
+        run: |
+          pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
+          pip install dist/*tar.gz
+      - name: Install dist w/o go
+        if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}}
+        run: pip install dist/*tar.gz
+      - name: Install OS X dependencies
+        if: matrix.os == 'macos-10.15'
+        run: brew install coreutils
+      - name: Smoke test
+        run: |
+          feast init test_repo
+          cd test_repo/
+          feast apply
+          echo "$TEST_SCRIPT" > run-and-wait.sh
+          bash run-and-wait.sh feast serve
+          bash run-and-wait.sh feast ui
+      # We disable this test for the Python 3.10 binary since it does not include Go.
+      - name: Smoke test with go
+        if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest'
+        run: |
+          cd test_repo/
+          feast apply
+          echo "$TEST_SCRIPT" > run-and-wait.sh
+          pip install cffi
+          printf "\ngo_feature_retrieval: True" >> feature_store.yaml
+          bash run-and-wait.sh feast serve

.github/workflows/publish.yml

@@ -138,78 +138,7 @@ jobs:
           user: __token__
           password: ${{ secrets.PYPI_PASSWORD }}
 
-
-  verify-python-wheel:
-    runs-on: ${{ matrix.os }}
-    needs: [build-python-sdk, build-python-sdk-macos-py310]
-    strategy:
-      matrix:
-        os: [ ubuntu-latest, macos-10.15 ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10"]
-        from-source: [ True, False ]
-    env:
-      # this script is for testing servers
-      # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
-      TEST_SCRIPT: |
-        timeout 10s $@ & pid=$!
-        wait $pid
-        ret=$?
-        if [[ $ret -ne 124 ]]
-        then
-           exit $ret
-        else
-           echo "Succeeded!"
-        fi
-    steps:
-      - name: Setup Python
-        id: setup-python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-      - uses: actions/setup-go@v3
-        with:
-          go-version: '>=1.17.0'
-      - uses: actions/download-artifact@v2
-        with:
-          name: wheels
-          path: dist
-      - name: Install wheel
-        if: ${{ !matrix.from-source }}
-        # try to install all wheels; only the current platform wheel should be actually installed
-        run: |
-          cd dist/
-          pip install wheel
-          for f in *.whl; do pip install $f || true; done
-      - name: Install sdist
-        if: ${{ matrix.from-source }}
-        env:
-          COMPILE_GO: "True"
-        run: |
-          pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
-          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
-          pip install dist/*tar.gz
-      - name: Install OS X dependencies
-        if: matrix.os == 'macos-10.15'
-        run: brew install coreutils
-      - name: Smoke test
-        run: |
-          feast init test_repo
-          cd test_repo/
-          feast apply
-          echo "$TEST_SCRIPT" > run-and-wait.sh
-          bash run-and-wait.sh feast serve
-          bash run-and-wait.sh feast ui
-      # We disable this test for the Python 3.10 binary since it does not include Go.
-      - name: Smoke test with go
-        if: matrix.python-version != '3.10'
-        run: |
-          pip install cffi
-          printf "\ngo_feature_retrieval: True" >> feature_store.yaml
-          bash run-and-wait.sh feast serve
-
-  build-python-sdk:
+  build-python-wheels:
     name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -225,7 +154,7 @@ jobs:
       - name: Build UI
         run: make build-ui
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.4.0
+        uses: pypa/cibuildwheel@v2.7.0
         env:
           CIBW_BUILD: "cp3*_x86_64"
           CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
@@ -243,14 +172,19 @@ jobs:
             make install-protoc-dependencies
             make install-go-proto-dependencies
             make install-go-ci-dependencies
-
+      - name: Build py310 specific wheels for macos
+        if: matrix.os == 'macos-10.15'
+        uses: pypa/cibuildwheel@v2.7.0
+        env:
+          CIBW_BUILD: "cp310-macosx_x86_64"
+          CIBW_ARCHS: "native"
       - uses: actions/upload-artifact@v2
         with:
           name: wheels
           path: ./wheelhouse/*.whl
 
-
-  build-python-sdk-macos-py310:
+  build-source-distribution:
+    name: Build source distribution
     runs-on: macos-10.15
     steps:
       - uses: actions/checkout@v2
@@ -274,13 +208,86 @@ jobs:
           make build-ui
       - name: Build
         run: |
-          python3 setup.py sdist bdist_wheel
-
+          python3 setup.py sdist
       - uses: actions/upload-artifact@v2
         with:
           name: wheels
           path: dist/*
 
+  verify-python-wheel:
+    runs-on: ${{ matrix.os }}
+    needs: [build-python-wheel, build-source-distribution]
+    strategy:
+      matrix:
+        os: [ubuntu-latest,  macos-10.15 ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10"]
+        from-source: [ True, False ]
+    env:
+      # this script is for testing servers
+      # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
+      TEST_SCRIPT: |
+        timeout 10s $@ & pid=$!
+        wait $pid
+        ret=$?
+        if [[ $ret -ne 124 ]]
+        then
+           exit $ret
+        else
+           echo "Succeeded!"
+        fi
+    steps:
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.17.0'
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+          path: dist
+      - name: Install wheel
+        if: ${{ !matrix.from-source }}
+        # try to install all wheels; only the current platform wheel should be actually installed
+        run: |
+          cd dist/
+          pip install wheel
+          for f in *.whl; do pip install $f || true; done
+      - name: Install go from sdist
+        if: ${{ matrix.from-source }}
+        env:
+          COMPILE_GO: "True"
+        run: |
+          pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
+      - name: Install dist
+        if: ${{ matrix.from-source }}
+        run: pip install dist/*tar.gz
+      - name: Install OS X dependencies
+        if: matrix.os == 'macos-10.15'
+        run: brew install coreutils
+      - name: Smoke test
+        run: |
+          feast init test_repo
+          cd test_repo/
+          feast apply
+          echo "$TEST_SCRIPT" > run-and-wait.sh
+          bash run-and-wait.sh feast serve
+          bash run-and-wait.sh feast ui
+      # We disable this test for the Python 3.10 binary since it does not include Go.
+      - name: Smoke test with go
+        if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest'
+        run: |
+          cd test_repo/
+          feast apply
+          echo "$TEST_SCRIPT" > run-and-wait.sh
+          pip install cffi
+          printf "\ngo_feature_retrieval: True" >> feature_store.yaml
+          bash run-and-wait.sh feast serve
 
   publish-java-sdk:
     container: maven:3.6-jdk-11