[python] Scikit-build-core driven wheel creation (#1454)

eclipse-ecal · May 7, 2024 · e78be41 · e78be41
1 parent 6a4bf8b
commit e78be41
Show file tree

Hide file tree

Showing 17 changed files with 269 additions and 486 deletions.
diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml
@@ -99,10 +99,6 @@ jobs:
       run: cmake --build . --config Release
       working-directory: ${{ runner.workspace }}/_build
 
-    - name: Build Python Wheel
-      run: cmake --build . --target create_python_wheel --config Release
-      working-directory: ${{ runner.workspace }}/_build
-
 #    - name: Build Documentation C
 #      run: cmake --build . --target documentation_c
 #      working-directory: ${{ runner.workspace }}/_build
@@ -125,9 +121,3 @@ jobs:
       with:
         name: macos-dmg
         path: ${{ runner.workspace }}/_build/_deploy/*.dmg
-
-    - name: Upload Python Wheel
-      uses: actions/upload-artifact@v4
-      with:
-        name: macos-python-wheel
-        path: ${{ runner.workspace }}/_build/_deploy/*.whl
diff --git a/.github/workflows/build-ubuntu-20.yml b/.github/workflows/build-ubuntu-20.yml
@@ -104,39 +104,6 @@ jobs:
       run: cmake --build . --config Release -- -k 0
       working-directory: ${{ runner.workspace }}/_build
 
-    # Create Python Wheels
-    # The strang-looking double-cmake is an ugly workaround to force CMake to
-    # re-find Python, after we have changed the venv from the outside. The
-    # alternative would be to clean everything, which would cause an unnecessary
-    # rebuild of eCAL for each python Version.
-    - name: Build Python 3.9 Wheel
-      run: |
-        sudo apt-get -y install python3.9-dev python3.9-venv
-        mkdir ".venv_39"
-        python3.9 -m venv ".venv_39"
-        source ".venv_39/bin/activate"
-        pip install --upgrade pip
-        pip install wheel setuptools
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=FIRST
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: bash
-      working-directory: ${{ runner.workspace }}/_build
-
-    - name: Build Python 3.8 Wheel
-      run: |
-        sudo apt-get -y install python3.8-dev python3.8-venv
-        mkdir ".venv_38"
-        python3.8 -m venv ".venv_38"
-        source ".venv_38/bin/activate"
-        pip install --upgrade pip
-        pip install wheel setuptools
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=FIRST
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: bash
-      working-directory: ${{ runner.workspace }}/_build
-
     - name: Run Tests
       run: ctest -V
       working-directory: ${{ runner.workspace }}/_build
@@ -151,8 +118,3 @@ jobs:
         name: ubuntu-debian
         path: ${{ runner.workspace }}/_build/_deploy/*.deb
 
-    - name: Upload Python Wheel
-      uses: actions/upload-artifact@v4
-      with:
-        name: ubuntu-python-wheel
-        path: ${{ runner.workspace }}/_build/_deploy/*.whl
diff --git a/.github/workflows/build-ubuntu-22.yml b/.github/workflows/build-ubuntu-22.yml
@@ -103,39 +103,6 @@ jobs:
       run: cmake --build . --config Release -- -k 0
       working-directory: ${{ runner.workspace }}/_build
 
-    # Create Python Wheels
-    # The strang-looking double-cmake is an ugly workaround to force CMake to
-    # re-find Python, after we have changed the venv from the outside. The
-    # alternative would be to clean everything, which would cause an unnecessary
-    # rebuild of eCAL for each python Version.
-    - name: Build Python 3.11 Wheel
-      run: |
-        sudo apt-get -y install python3.11-dev python3.11-venv
-        mkdir ".venv_311"
-        python3.11 -m venv ".venv_311"
-        source ".venv_311/bin/activate"
-        pip install --upgrade pip
-        pip install wheel setuptools
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=FIRST
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: bash
-      working-directory: ${{ runner.workspace }}/_build
-
-    - name: Build Python 3.10 Wheel
-      run: |
-        sudo apt-get -y install python3.10-dev python3.10-venv
-        mkdir ".venv_310"
-        python3.10 -m venv ".venv_310"
-        source ".venv_310/bin/activate"
-        pip install --upgrade pip
-        pip install wheel setuptools
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=FIRST
-        cmake $GITHUB_WORKSPACE -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: bash
-      working-directory: ${{ runner.workspace }}/_build
-
     - name: Run Tests
       run: ctest -V
       working-directory: ${{ runner.workspace }}/_build
@@ -149,9 +116,3 @@ jobs:
       with:
         name: ubuntu-debian
         path: ${{ runner.workspace }}/_build/_deploy/*.deb
-
-    - name: Upload Python Wheel
-      uses: actions/upload-artifact@v4
-      with:
-        name: ubuntu-python-wheel
-        path: ${{ runner.workspace }}/_build/_deploy/*.whl
diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
@@ -127,7 +127,7 @@ jobs:
         -DBUILD_APPS=ON ^
         -DBUILD_SAMPLES=ON ^
         -DBUILD_TIME=ON ^
-        -DBUILD_PY_BINDING=ON ^
+        -DBUILD_PY_BINDING=OFF ^
         -DBUILD_CSHARP_BINDING=ON ^
         -DBUILD_ECAL_TESTS=ON ^
         -DECAL_INCLUDE_PY_SAMPLES=OFF ^
@@ -163,83 +163,6 @@ jobs:
       run: cmake --build . --config Release
       working-directory: ${{ runner.workspace }}/_build/complete
 
-    # Create Python.
-    # The strang-looking double-cmake is an ugly workaround to force CMake to
-    # re-find Python, after we have changed the venv from the outside. The
-    # alternative would be to clean everything, which would cause an unnecessary
-    # rebuild of eCAL and HDF5 for each python Version.
-    - name: Build Python 3.12 Wheel
-      run: |
-        mkdir ".venv_312"
-        py -3.12 -m venv ".venv_312"
-        CALL ".venv_312\Scripts\activate.bat"
-        pip install wheel  setuptools
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=FIRST
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: cmd
-      working-directory: ${{ runner.workspace }}/_build/complete
-
-    - name: Build Python 3.11 Wheel
-      run: |
-        mkdir ".venv_311"
-        py -3.11 -m venv ".venv_311"
-        CALL ".venv_311\Scripts\activate.bat"
-        pip install wheel
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=FIRST
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: cmd
-      working-directory: ${{ runner.workspace }}/_build/complete
-
-    - name: Build Python 3.10 Wheel
-      run: |
-        mkdir ".venv_310"
-        py -3.10 -m venv ".venv_310"
-        CALL ".venv_310\Scripts\activate.bat"
-        pip install wheel
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=FIRST
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: cmd
-      working-directory: ${{ runner.workspace }}/_build/complete
-
-    - name: Build Python 3.9 Wheel
-      run: |
-        mkdir ".venv_39"
-        py -3.9 -m venv ".venv_39"
-        CALL ".venv_39\Scripts\activate.bat"
-        pip install wheel
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=FIRST
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: cmd
-      working-directory: ${{ runner.workspace }}/_build/complete
-
-    - name: Build Python 3.8 Wheel
-      run: |
-        mkdir ".venv_38"
-        py -3.8 -m venv ".venv_38"
-        CALL ".venv_38\Scripts\activate.bat"
-        pip install wheel
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=FIRST
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: cmd
-      working-directory: ${{ runner.workspace }}/_build/complete
-
-    - name: Build Python 3.7 Wheel
-      run: |
-        mkdir ".venv_37"
-        py -3.7 -m venv ".venv_37"
-        CALL ".venv_37\Scripts\activate.bat"
-        pip install wheel
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=FIRST
-        cmake %GITHUB_WORKSPACE% -G "Visual Studio 16 2019" -A x64 -T v142 -DPython_FIND_VIRTUALENV=ONLY
-        cmake --build . --target create_python_wheel --config Release
-      shell: cmd
-      working-directory: ${{ runner.workspace }}/_build/complete
-
 #    - name: Build Documentation C
 #      run: cmake --build . --target documentation_c
 #      working-directory: ${{ runner.workspace }}/_build
@@ -266,12 +189,6 @@ jobs:
         name: windows-setup
         path: ${{ runner.workspace }}/_build/complete/_deploy/*.exe
 
-    - name: Upload Python Wheels
-      uses: actions/upload-artifact@v4
-      with:
-        name: windows-python-wheels
-        path: ${{ runner.workspace }}/_build/complete/_deploy/*.whl
-
     # --------------------------------------------------------------------------------------------------
 
   sign-windows-installer:

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -0,0 +1,63 @@
+name: Build Wheels
+
+on:
+  pull_request:
+    branches:
+      - master
+  release:
+    types: [published]
+  workflow_dispatch:
+
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules:  'true'
+          fetch-depth: 0
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.17
+        env:
+          CIBW_ARCHS: auto64
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
+  build_arm64_wheels:
+    # Emulation takes a long time so we save it for release/manual triggering
+    name: Build arm64 wheels via emulation
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules:  'true'
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.17
+        env:
+          CIBW_ARCHS_LINUX: aarch64
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-arm64
+          path: ./wheelhouse/*.whl
+
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,12 @@ Thumbs.db
 /_vs_out*
 /out
 
+# Python building and distribution
+/_python_build/
+/dist/
+/lang/python/core/ecal/_version.py
+/wheelhouse/
+
 # Binary libraries
 /thirdparty/npcap/
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -88,7 +88,6 @@ option(BUILD_APPS                              "Build the eCAL applications"
 option(BUILD_SAMPLES                           "Build the eCAL samples"                                            ON)
 option(BUILD_TIME                              "Build the eCAL time interfaces"                                    ON)
 option(BUILD_PY_BINDING                        "Build eCAL python binding"                                        OFF)
-option(BUILD_STANDALONE_PY_WHEEL               "Build eCAL python binding as standalone wheel"                    OFF)
 option(BUILD_CSHARP_BINDING                    "Build eCAL C# binding"                                            OFF)
 option(BUILD_ECAL_TESTS                        "Build the eCAL google tests"                                      OFF)
 
@@ -125,7 +124,18 @@ option(CPACK_PACK_WITH_INNOSETUP               "Create Innosetup installer for t
 
 set(ECAL_INSTALL_PYTHON_DIR "bin" CACHE PATH "Location to install the Python extension modules. Might be set by setupdtools install.")
 
-set(ECAL_BUILD_VERSION "0.0.0" CACHE STRING "Inject a build version if not building from a git repository")
+if(DEFINED SKBUILD_PROJECT_VERSION)
+  message(STATUS
+    "Using version from scikit-build-core: ${SKBUILD_PROJECT_VERSION}"
+  )
+  set(ECAL_BUILD_VERSION "${SKBUILD_PROJECT_VERSION}" CACHE STRING
+    "Version provided by scikit-build-core and setuptools-scm" FORCE
+  )
+else()
+  set(ECAL_BUILD_VERSION "0.0.0" CACHE STRING
+    "Inject a build version if not building from a git repository"
+  )
+endif()
 
 set(ECAL_CSHARP_BUILD_SAMPLES  ${BUILD_SAMPLES})
 
@@ -172,10 +182,12 @@ set(eCAL_VERSION_PATCH  ${GIT_REVISION_PATCH})
 set(eCAL_VERSION_STRING ${eCAL_VERSION_MAJOR}.${eCAL_VERSION_MINOR}.${eCAL_VERSION_PATCH})
 set(eCAL_VERSION        ${eCAL_VERSION_STRING})
 
+message(STATUS "eCAL version: ${eCAL_VERSION_STRING}")
+message(DEBUG "eCAL git describe tag: ${GIT_DESCRIBE_TAG}")
+
 include(helper_functions/ecal_add_functions)
 include(helper_functions/ecal_helper_functions)
 include(helper_functions/ecal_install_functions)
-include(helper_functions/ecal_python_functions)
 
 if(MSVC)
   set(eCAL_PLATFORM_TOOLSET ${CMAKE_VS_PLATFORM_TOOLSET})
@@ -465,7 +477,6 @@ message(STATUS "BUILD_APPS                                     : ${BUILD_APPS}")
 message(STATUS "BUILD_SAMPLES                                  : ${BUILD_SAMPLES}")
 message(STATUS "BUILD_TIME                                     : ${BUILD_TIME}")
 message(STATUS "BUILD_PY_BINDING                               : ${BUILD_PY_BINDING}")
-message(STATUS "BUILD_STANDALONE_PY_WHEEL                      : ${BUILD_STANDALONE_PY_WHEEL}")
 message(STATUS "BUILD_CSHARP_BINDING                           : ${BUILD_CSHARP_BINDING}")
 message(STATUS "BUILD_ECAL_TESTS                               : ${BUILD_ECAL_TESTS}")
 message(STATUS "ECAL_INCLUDE_PY_SAMPLES                        : ${ECAL_INCLUDE_PY_SAMPLES}")