update release ci (#4359)

* update release ci * find modern glslang * parallel jobs on windows
Tencent · Nov 20, 2022 · 6647396 · 6647396
1 parent a5e60ae
commit 6647396
Show file tree

Hide file tree

Showing 15 changed files with 134 additions and 72 deletions.
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
@@ -1,4 +1,5 @@
 name: release-python
+# on: [push, pull_request]
 on:
   push:
     tags:
@@ -31,12 +32,26 @@ jobs:
         path: dist/*.tar.gz
 
   build_wheels:
-    name: Build wheels on ${{ matrix.os }}
+    name: ${{ matrix.arch }} ${{ matrix.build }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-11]
+        include:
+          - { os: ubuntu-20.04, arch: x86_64,     build: 'cp*-manylinux*' }
+          - { os: ubuntu-20.04, arch: x86_64,     build: 'cp*-musllinux*' }
+          - { os: ubuntu-20.04, arch: x86_64,     build: 'pp*'            }
+          - { os: ubuntu-20.04, arch: i686,       build: 'cp*-manylinux*' }
+          - { os: ubuntu-20.04, arch: i686,       build: 'cp*-musllinux*' }
+          - { os: ubuntu-20.04, arch: i686,       build: 'pp*'            }
+          - { os: windows-2019, arch: x86,        build: 'cp*'            }
+          - { os: windows-2019, arch: AMD64,      build: 'cp*'            }
+          - { os: windows-2019, arch: AMD64,      build: 'pp*'            }
+          - { os: windows-2019, arch: ARM64,      build: 'cp*'            }
+          - { os: macos-11,     arch: x86_64,     build: 'cp*'            }
+          - { os: macos-11,     arch: x86_64,     build: 'pp*'            }
+          - { os: macos-11,     arch: arm64,      build: 'cp*'            }
+          - { os: macos-11,     arch: universal2, build: 'cp*'            }
 
     steps:
     - uses: actions/checkout@v3
@@ -55,8 +70,12 @@ jobs:
     - name: Build wheels
       uses: pypa/cibuildwheel@v2.11.2
       env:
-        CIBW_ARCHS_MACOS: x86_64 universal2 arm64
-        CIBW_ARCHS_LINUX: x86_64 i686
+        CIBW_ARCHS_MACOS: ${{ matrix.arch }}
+        CIBW_ARCHS_LINUX: ${{ matrix.arch }}
+        CIBW_ARCHS_WINDOWS: ${{ matrix.arch }}
+        CIBW_BUILD: ${{ matrix.build }}
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=2
 
     - name: Show files
       run: ls -lh wheelhouse
@@ -72,21 +91,21 @@ jobs:
         path: wheelhouse/*.whl
 
   build_wheels_qemu:
-    name: Build wheels ${{ matrix.arch }} ${{ matrix.build }}
+    name: ${{ matrix.arch }} ${{ matrix.build }}
     runs-on: ubuntu-20.04
 
     strategy:
       fail-fast: false
       matrix:
         arch: [aarch64, ppc64le, s390x]
-        build: ["cp36-*", "cp37-*", "cp38-*", "cp39-*", "cp310-*"]
+        build: ['cp36-*', 'cp37-*', 'cp38-*', 'cp39-*', 'cp310-*', 'cp311-*']
         include:
           - arch: aarch64
-            build: "pp37-*"
+            build: 'pp37-*'
           - arch: aarch64
-            build: "pp38-*"
+            build: 'pp38-*'
           - arch: aarch64
-            build: "pp39-*"
+            build: 'pp39-*'
 
     steps:
     - uses: actions/checkout@v3
@@ -107,6 +126,8 @@ jobs:
       env:
         CIBW_ARCHS_LINUX: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build }}
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=2
 
     - name: Show files
       run: ls -lh wheelhouse
@@ -138,7 +159,7 @@ jobs:
         name: artifact
         path: dist
 
-    - uses: pypa/gh-action-pypi-publish@v1.5.1
+    - uses: pypa/gh-action-pypi-publish@v1
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
     - name: get-version
       id: get_version
-      run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
 
   full-source:
     needs: [setup]
@@ -237,7 +237,7 @@ jobs:
         path: ${{ env.PACKAGENAME }}.zip
 
   openmp-macos:
-    runs-on: macos-latest
+    runs-on: macos-11
     steps:
     - name: cache-openmp
       id: cache-openmp
@@ -290,7 +290,7 @@ jobs:
 
   macos:
     needs: [setup, openmp-macos]
-    runs-on: macos-latest
+    runs-on: macos-11
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-macos
     steps:
@@ -358,7 +358,7 @@ jobs:
 
   macos-gpu:
     needs: [setup, openmp-macos]
-    runs-on: macos-latest
+    runs-on: macos-11
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-macos-vulkan
     steps:
@@ -454,7 +454,7 @@ jobs:
         path: ${{ env.PACKAGENAME }}.zip
 
   openmp-ios:
-    runs-on: macos-latest
+    runs-on: macos-11
     steps:
     - name: cache-openmp
       id: cache-openmp
@@ -511,7 +511,7 @@ jobs:
 
   ios:
     needs: [setup, openmp-ios]
-    runs-on: macos-latest
+    runs-on: macos-11
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-ios
     steps:
@@ -594,7 +594,7 @@ jobs:
 
   ios-gpu:
     needs: [setup, openmp-ios]
-    runs-on: macos-latest
+    runs-on: macos-11
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-ios-vulkan
     steps:
@@ -693,7 +693,7 @@ jobs:
         path: ${{ env.PACKAGENAME }}.zip
 
   openmp-ios-bitcode:
-    runs-on: macos-latest
+    runs-on: macos-11
     steps:
     - name: cache-openmp
       id: cache-openmp
@@ -750,7 +750,7 @@ jobs:
 
   ios-bitcode:
     needs: [setup, openmp-ios-bitcode]
-    runs-on: macos-latest
+    runs-on: macos-11
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-ios-bitcode
     steps:
@@ -833,7 +833,7 @@ jobs:
 
   ios-gpu-bitcode:
     needs: [setup, openmp-ios-bitcode]
-    runs-on: macos-latest
+    runs-on: macos-11
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-ios-vulkan-bitcode
     steps:
@@ -1267,6 +1267,7 @@ jobs:
     runs-on: windows-2019
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2015
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1332,6 +1333,7 @@ jobs:
     runs-on: windows-2019
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2015-shared
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1397,6 +1399,7 @@ jobs:
     runs-on: windows-2019
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2017
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1462,6 +1465,7 @@ jobs:
     runs-on: windows-2019
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2017-shared
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1527,6 +1531,7 @@ jobs:
     runs-on: windows-latest
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2019
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1608,6 +1613,7 @@ jobs:
     runs-on: windows-latest
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2019-shared
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1689,6 +1695,7 @@ jobs:
     runs-on: windows-latest
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2022
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:
@@ -1770,6 +1777,7 @@ jobs:
     runs-on: windows-latest
     env:
       PACKAGENAME: ncnn-${{ needs.setup.outputs.VERSION }}-windows-vs2022-shared
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:

diff --git a/.github/workflows/windows-arm-cpu.yml b/.github/workflows/windows-arm-cpu.yml
@@ -41,6 +41,8 @@ jobs:
             toolset-version: v143
             os: windows-2022
 
+    env:
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
     - name: build

diff --git a/.github/workflows/windows-arm64-cpu.yml b/.github/workflows/windows-arm64-cpu.yml
@@ -41,6 +41,8 @@ jobs:
             toolset-version: v143
             os: windows-2022
 
+    env:
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
     - name: build

diff --git a/.github/workflows/windows-x64-cpu-vs2019-python.yml b/.github/workflows/windows-x64-cpu-vs2019-python.yml
@@ -32,6 +32,8 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
+    env:
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:

diff --git a/.github/workflows/windows-x64-cpu.yml b/.github/workflows/windows-x64-cpu.yml
@@ -55,6 +55,8 @@ jobs:
             toolset-version: v143
             os: windows-2022
 
+    env:
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
     - name: cache-protobuf

diff --git a/.github/workflows/windows-x64-gpu.yml b/.github/workflows/windows-x64-gpu.yml
@@ -57,6 +57,8 @@ jobs:
             toolset-version: v143
             os: windows-2022
 
+    env:
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
       with:

diff --git a/.github/workflows/windows-x86-cpu.yml b/.github/workflows/windows-x86-cpu.yml
@@ -49,6 +49,8 @@ jobs:
             toolset-version: v143
             os: windows-2022
 
+    env:
+      UseMultiToolTask: true
     steps:
     - uses: actions/checkout@v3
     - name: build

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -496,29 +496,38 @@ endif()
 
 if(NCNN_VULKAN)
     if(NCNN_SYSTEM_GLSLANG)
-        set(GLSLANG_TARGET_DIR "GLSLANG-NOTFOUND" CACHE PATH "Absolute path to glslangTargets.cmake directory")
-        if(NOT GLSLANG_TARGET_DIR AND NOT DEFINED ENV{GLSLANG_TARGET_DIR})
-            message(WARNING "GLSLANG_TARGET_DIR must be defined! NCNN_SYSTEM_GLSLANG will be turned off.")
-            set(NCNN_SYSTEM_GLSLANG OFF)
+        find_package(Threads)
+        find_package(glslang QUIET)
+        if(glslang_FOUND)
+            add_library(glslang ALIAS glslang::glslang)
+            add_library(SPIRV ALIAS glslang::SPIRV)
         else()
-            message(STATUS "Using glslang install located at ${GLSLANG_TARGET_DIR}")
-
-            find_package(Threads)
-
-            include("${GLSLANG_TARGET_DIR}/OSDependentTargets.cmake")
-            include("${GLSLANG_TARGET_DIR}/OGLCompilerTargets.cmake")
-            if(EXISTS "${GLSLANG_TARGET_DIR}/HLSLTargets.cmake")
-                # hlsl support can be optional
-                include("${GLSLANG_TARGET_DIR}/HLSLTargets.cmake")
-            endif()
-            include("${GLSLANG_TARGET_DIR}/glslangTargets.cmake")
-            include("${GLSLANG_TARGET_DIR}/SPIRVTargets.cmake")
-
-            if (NOT TARGET glslang OR NOT TARGET SPIRV)
-                message(WARNING "glslang or SPIRV target not found! NCNN_SYSTEM_GLSLANG will be turned off.")
+            set(GLSLANG_TARGET_DIR "GLSLANG-NOTFOUND" CACHE PATH "Absolute path to glslangTargets.cmake directory")
+            if(NOT GLSLANG_TARGET_DIR AND NOT DEFINED ENV{GLSLANG_TARGET_DIR})
+                message(WARNING "set glslang_DIR to glslang-config.cmake directory for using system glslang.")
+                message(WARNING "GLSLANG_TARGET_DIR must be defined! NCNN_SYSTEM_GLSLANG will be turned off.")
                 set(NCNN_SYSTEM_GLSLANG OFF)
+            else()
+                include("${GLSLANG_TARGET_DIR}/OSDependentTargets.cmake")
+                include("${GLSLANG_TARGET_DIR}/OGLCompilerTargets.cmake")
+                if(EXISTS "${GLSLANG_TARGET_DIR}/HLSLTargets.cmake")
+                    # hlsl support can be optional
+                    include("${GLSLANG_TARGET_DIR}/HLSLTargets.cmake")
+                endif()
+                include("${GLSLANG_TARGET_DIR}/glslangTargets.cmake")
+                include("${GLSLANG_TARGET_DIR}/SPIRVTargets.cmake")
             endif()
         endif()
+
+        if (TARGET glslang AND TARGET SPIRV)
+            get_property(glslang_location TARGET glslang PROPERTY LOCATION)
+            get_property(SPIRV_location TARGET SPIRV PROPERTY LOCATION)
+            message(STATUS "Found glslang: ${glslang_location} (found version \"${glslang_VERSION}\")")
+            message(STATUS "Found SPIRV: ${SPIRV_location} (found version \"${glslang_VERSION}\")")
+        else()
+            message(WARNING "glslang or SPIRV target not found! NCNN_SYSTEM_GLSLANG will be turned off.")
+            set(NCNN_SYSTEM_GLSLANG OFF)
+        endif()
     endif()
 
     if(NOT NCNN_SYSTEM_GLSLANG)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -8,6 +8,19 @@ add_definitions(-DVERSION_INFO="${PACKAGE_VERSION}")
 set( CMAKE_CXX_STANDARD 11 )
 set( CMAKE_CXX_STANDARD_REQUIRED ON )
 
+if(CMAKE_CXX_COMPILER_ARCHITECTURE_ID MATCHES "ARM64")
+    option(PYBIND11_PYTHONLIBS_OVERWRITE "" OFF)
+
+    set(PYTHON_PREFIX "$ENV{LOCALAPPDATA}/pypa/cibuildwheel/Cache/nuget-cpython/pythonarm64.$ENV{PYTHON_VERSION}/tools")
+    if(NOT DEFINED $ENV{CIBUILDWHEEL})
+        message(WARNING
+            " This is hack for cibuildwheel on github action\n"
+            " Use the right way to cross-compile python module for windows arm64 like follows\n"
+            " set(PYTHON_PREFIX \"<your-pythonarm64-root-path>\")\n"
+        )
+    endif()
+endif()
+
 add_subdirectory(pybind11)
 
 if("${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" STREQUAL "")

diff --git a/python/pybind11 b/python/pybind11