move to Cibuildwheel workflow (#274)

switch the build system for python to cibuildwheel. cibuildwheel makes it easier to build python wheels on different platforms and python versions.

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/keyvi.yml

@@ -36,7 +36,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: ccache
-        uses: hendrikmuhs/ccache-action@v1.1
+        uses: hendrikmuhs/ccache-action@v1.2.11
         with:
           key: ${{ matrix.os }}-${{ matrix.type }}
 

.github/workflows/python-cibuildwheel.yml

@@ -0,0 +1,183 @@
+name: Python cibuildwheel
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04, macos-12]
+        # separate archs, so they use individual caches
+        arch: [ 'x86_64', 'arm64' ]
+        flavor: ['cpython', 'pypy']
+        # separate musl and many on linux, for mac we just skip one of those
+        target: [ 'many', 'musl' ]
+        exclude:
+          - os: macos-12
+            target: musl
+          - os: ubuntu-22.04
+            target: musl
+            flavor: pypy
+          - os: macos-12
+            arch: arm64
+            flavor: pypy
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }}
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.11
+        with:
+          key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python
+
+      - name: Sets env for x86_64
+        run: |
+          echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV
+          echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
+        if: matrix.arch == 'x86_64'
+
+      - name: Sets env for arm64
+        run: |
+          echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV
+          echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV
+        if: matrix.arch == 'arm64'
+
+      - name: Skip manylinux for musllinux target
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
+        run: |
+          echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV
+
+      - name: Skip musllinux for manylinux target
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
+        run: |
+          echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV
+
+      - name: Skip pypy for cpython
+        if: ${{ matrix.flavor == 'cpython' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV
+
+      - name: Skip cpython for pypy
+        if: ${{ matrix.flavor == 'pypy' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV
+
+      - name: install mac dependencies
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          brew update && \
+          brew install ccache
+
+      - name: install mac dependencies X86_64
+        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }}
+        run: |
+          brew update && \
+          brew install zlib snappy boost
+
+      - name: install mac dependencies arm64
+        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }}
+        run: |
+          set -e
+          echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV
+          echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV
+          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV
+          mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew
+          PACKAGES=( icu4c xz lz4 zstd zlib snappy boost )
+          for PACKAGE in "${PACKAGES[@]}"
+          do
+            response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_monterey $PACKAGE | grep Downloaded )
+            download_path=$(echo $response | xargs -n 1 | tail -1)
+            arm64-homebrew/bin/brew reinstall -vd $download_path
+          done
+          arm64-homebrew/bin/brew config
+          ls /Users/runner/work/keyvi/keyvi/arm64-homebrew
+
+      - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }}
+        uses: pypa/cibuildwheel@v2.16.2
+        env:
+          # Skip CPython 3.6 and CPython 3.7
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-*
+
+          # skip testing all python versions on linux arm, only test 3.12
+          # skip tests on pypy, currently fails for indexer tests
+          CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*"
+
+          # (many)linux custom docker images
+          CIBW_MANYLINUX_X86_64_IMAGE: 'keyvidev/manylinux-builder-x86_64'
+          CIBW_MANYLINUX_AARCH64_IMAGE: 'keyvidev/manylinux-builder-aarch64'
+          CIBW_MUSLLINUX_X86_64_IMAGE: 'keyvidev/musllinux-builder-x86_64'
+          CIBW_MUSLLINUX_AARCH64_IMAGE: 'keyvidev/musllinux-builder-aarch64'
+
+          # ccache using path
+          CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH
+          CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf
+
+          # python dependencies
+          CIBW_BEFORE_BUILD: pip install -r python/requirements.txt
+
+          # testing
+          CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_COMMAND: >
+            python -m pytest {package}/tests &&
+            python -m pytest {package}/integration-tests
+
+          # for debugging set this to 1,2 or 3
+          # CIBW_BUILD_VERBOSITY: 2
+        with:
+          package-dir: python
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: install Linux deps
+      run: |
+        sudo apt-get update && \
+        sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache
+    - name: ccache
+      uses: hendrikmuhs/ccache-action@v1.2.11
+      with:
+        key: ${{ matrix.os }}-sdist-python
+
+    - name: Build SDist
+      run: |
+        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+        cd python && \
+        python -m pip install -r requirements.txt && \
+        python setup.py sdist -d wheelhouse && \
+        python -m pip uninstall -y autowrap && \
+        python -m pip install wheelhouse/*.tar.gz -v && \
+        python -m pytest tests && \
+        python -m pip uninstall -y keyvi
+
+    - uses: actions/upload-artifact@v3
+      with:
+        path: python/wheelhouse/*.tar.gz
+
+  upload_all:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.pypi_password }}

CMakeLists.txt

@@ -1,9 +1,6 @@
 cmake_minimum_required(VERSION 3.9)
 project(keyvi)
 
-EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE )
-message( STATUS "Architecture: ${ARCHITECTURE}" )
-
 #### Build Type
 if (CMAKE_BUILD_TYPE)
   string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
@@ -22,9 +19,9 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 set (_KEVYI_COMPILE_OPTIONS "-Wall")
 set (_KEYVI_COMPILE_DEFINITIONS "RAPIDJSON_HAS_STDSTRING")
 
-if(${ARCHITECTURE} STREQUAL "x86_64")
+if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     set (_KEYVI_CXX_FLAGS "-msse4.2")
-    message( STATUS "Architecture: ${ARCHITECTURE} detected: added -msse4.2 compile flag")
+    message( STATUS "Architecture: ${CMAKE_SYSTEM_PROCESSOR} detected: added -msse4.2 compile flag")
 endif()
 
 set (_OS_LIBRARIES "")