fix windows?

nod-ai · May 15, 2024 · ee7301c · ee7301c
1 parent 3536701
commit ee7301c
Show file tree

Hide file tree

Showing 14 changed files with 980 additions and 87 deletions.
diff --git a/.github/workflows/base/action.yml b/.github/workflows/base/action.yml
@@ -7,13 +7,19 @@ inputs:
   ARCH:
     description: 'matrix.arch'
     required: true
+  PY_VERSION:
+    description: 'matrix.py_version'
+    required: true
 
 description: ''
 
 outputs:
   BOOST_ROOT:
     description: ''
     value: ${{ steps.install_boost.outputs.BOOST_ROOT }}
+  BOOST_VERSION:
+    description: ''
+    value: 1.74.0
 
 runs:
   using: "composite"
@@ -22,12 +28,16 @@ runs:
       uses: aminya/setup-cpp@v1
       with:
         compiler: gcc
-        vcvarsall: ${{ contains(inputs.OS, 'windows') }}
+        vcvarsall: ${{ contains(inputs.OS, 'windows') }} # evaluate to the string 'true'
         msvc: ${{ contains(inputs.OS, 'windows') }}
         choco: ${{ contains(inputs.OS, 'windows') }}
         cmake: true
         ninja: true
 
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.PY_VERSION }}
+
     - uses: makslevental/mlir-wheels/actions/setup_ccache@d77bf5dc69c46a8c2738b44528749768888eb361
       id: setup_ccache
       with:
@@ -36,8 +46,36 @@ runs:
         WORKSPACE_ROOT: ${{ github.workspace }}
 
     - name: Install boost for XRT
-      if: contains(inputs.OS, 'windows') || contains(inputs.OS, 'ubuntu')
       uses: MarkusJx/install-boost@v2.4.5
       id: install_boost
       with:
-        boost_version: 1.73.0
+        boost_version: 1.74.0
+
+    - name: Build and install system deps
+      shell: bash
+      run: |
+        
+        if [ ${{ inputs.OS }} == 'windows-2019' ]; then
+          choco install -y pkgconfiglite protoc 
+        else
+          sudo apt install -y libdrm-dev ocl-icd-opencl-dev rapidjson-dev libprotobuf-dev systemtap-sdt-dev uuid-dev python3-dev
+        fi
+
+
+
+    - name: Remove path limit and export VCTOOLSVERSION
+      if: contains(inputs.OS, 'windows')
+      shell: pwsh
+      run: |
+        
+        pwsh.exe .\util\remove_path_limit.ps1
+
+        VCTOOLSVERSION=${VCTOOLSVERSION:0:4}
+        echo "VCTOOLSVERSION=$VCTOOLSVERSION" >> $GITHUB_ENV
+
+    - name: Install python deps
+      shell: bash
+      run: |
+        
+        ls -lah
+        pip install -r requirements-dev.txt
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -28,12 +28,13 @@ jobs:
         shell: bash
 
     steps:
+
       - name: Checkout actions
         uses: actions/checkout@v3
         with:
           submodules: true
 
-      - name: Get recursive submodules
+      - name: Get Submodules
         run: |
           
           git submodule update --init --recursive
@@ -43,30 +44,18 @@ jobs:
         with:
           OS: ${{ matrix.OS }}
           ARCH: ${{ matrix.ARCH }}
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.PY_VERSION }}
-
-      - name: Build and install protoc
-        if: contains(matrix.OS, 'windows') || contains(matrix.OS, 'ubuntu')
-        run: |
-          
-          if [ ${{ matrix.OS }} == 'windows-2019' ]; then
-            choco install -y pkgconfiglite protoc 
-          else
-            sudo apt install -y libdrm-dev ocl-icd-opencl-dev rapidjson-dev libprotobuf-dev systemtap-sdt-dev uuid-dev python3-dev
-          fi
+          PY_VERSION: ${{ matrix.PY_VERSION }}
 
       - name: build wheel
         env:
           BOOST_ROOT: ${{ steps.base.outputs.BOOST_ROOT }}
-          BOOST_VERSION: 1.73.0
+          BOOST_VERSION: ${{ steps.base.outputs.BOOST_VERSION }}
         run: |
           
-          REPO_SRC_DIR=$PWD bash patches/apply_patches.sh
-          pip install -r requirements-dev.txt
-          pip wheel . -v -w wheelhouse --no-build-isolation
+          CIBW_BUILD=${{ matrix.PY_VERSION }}
+          CIBW_BUILD="cp${CIBW_BUILD//./}-*"
+          echo "CIBW_BUILD=$CIBW_BUILD" | tee -a $GITHUB_OUTPUT
+          cibuildwheel --output-dir wheelhouse
 
       - name: test
         run: |
@@ -99,52 +88,39 @@ jobs:
         PY_VERSION: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
+
       - name: Checkout actions
         uses: actions/checkout@v3
         with:
           submodules: true
 
-      - name: Get recursive submodules
+      - name: Get Submodules
         run: |
           
           git submodule update --init --recursive
 
-      - name: Setup Cpp
-        uses: aminya/setup-cpp@v1
-        with:
-          compiler: gcc
-          cmake: true
-          ninja: true
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:
           platforms: aarch64
 
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.PY_VERSION }}
-
       - uses: ./.github/workflows/base
         id: base
         with:
           OS: ${{ matrix.OS }}
           ARCH: ${{ matrix.ARCH }}
-
-      - name: Install XRT deps
-        run: |
-          
-          sudo apt install -y libdrm-dev ocl-icd-opencl-dev rapidjson-dev libprotobuf-dev systemtap-sdt-dev uuid-dev python3-dev
+          PY_VERSION: ${{ matrix.PY_VERSION }}
 
       - name: build wheel
         env:
           BOOST_ROOT: ${{ steps.base.outputs.BOOST_ROOT }}
-          BOOST_VERSION: 1.73.0
+          BOOST_VERSION: ${{ steps.base.outputs.BOOST_VERSION }}
         run: |
           
-          REPO_SRC_DIR=$PWD bash patches/apply_patches.sh
-          pip install -r requirements-dev.txt
-          pip wheel . -v -w wheelhouse --no-build-isolation
+          CIBW_BUILD=${{ matrix.PY_VERSION }}
+          CIBW_BUILD="cp${CIBW_BUILD//./}-*"
+          echo "CIBW_BUILD=$CIBW_BUILD" | tee -a $GITHUB_OUTPUT
+          cibuildwheel --output-dir wheelhouse
 
       - name: test
         run: |

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -40,33 +40,26 @@ jobs:
         with:
           submodules: true
 
-      - name: Get recursive submodules
+      - name: Get Submodules
         run: |
           
           git submodule update --init --recursive
 
       - uses: ./.github/workflows/base
-        if: contains(matrix.OS, 'windows')
         id: base
         with:
           OS: ${{ matrix.OS }}
           ARCH: ${{ matrix.ARCH }}
-
-      - name: Build and install protoc
-        if: contains(matrix.OS, 'windows')
-        run: |
-          
-          choco install -y pkgconfiglite protoc 
+          PY_VERSION: '3.10'
 
       # build
 
       - name: cibuildwheel python bindings
         env:
           BOOST_ROOT: ${{ steps.base.outputs.BOOST_ROOT }}
-          BOOST_VERSION: 1.73.0
+          BOOST_VERSION: ${{ steps.base.outputs.BOOST_VERSION }}
         run: |
 
-          pip install cibuildwheel
           cibuildwheel --output-dir wheelhouse
 
       # done
@@ -85,30 +78,36 @@ jobs:
 
     steps:
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: aarch64
+
       - name: Checkout actions
         uses: actions/checkout@v3
         with:
           submodules: true
 
-      - name: Get recursive submodules
+      - name: Get Submodules
         run: |
           
           git submodule update --init --recursive
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - uses: ./.github/workflows/base
+        id: base
         with:
-          platforms: aarch64
+          OS: ubuntu-20.04
+          ARCH: aarch64
+          PY_VERSION: '3.10'
 
       # build
 
       - name: cibuildwheel python bindings aarch64
         env:
           BOOST_ROOT: ${{ steps.base.outputs.BOOST_ROOT }}
-          BOOST_VERSION: 1.73.0
+          BOOST_VERSION: ${{ steps.base.outputs.BOOST_VERSION }}
         run: |
           
-          pip install cibuildwheel
           cibuildwheel --output-dir wheelhouse
 
       # done

diff --git a/.gitignore b/.gitignore
@@ -159,6 +159,18 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
-cmake-build-debug
-cmake-build-release
-.envrc
+cmake-build-*
+.envrc
+*.whl
+*.zip
+*.ilk
+*.pyd
+*.lib
+*.exp
+*.pdb
+*.dll
+*.json
+*.pdi
+*.bif
+*.bin
+*.xclbin
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -12,14 +12,43 @@ if(POLICY CMP0148)
   cmake_policy(SET CMP0148 NEW)
 endif()
 
-option(OUTPUT_DIR "Output directory" ${CMAKE_CURRENT_BINARY_DIR})
+if(WIN32)
+  # see util/remove_path_limit.ps1
+  set(CMAKE_OBJECT_PATH_MAX 4096)
+endif()
 
-include(util/helpers.cmake)
+set(OUTPUT_DIR
+    ${CMAKE_CURRENT_BINARY_DIR}
+    CACHE STRING "Output directory")
+
+if(NOT ${CMAKE_SIZEOF_VOID_P} EQUAL 8)
+  # if you open up the wrong developer shell in windows...
+  message(
+    FATAL_ERROR
+      "Something has gone terribly wrong: CMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P}"
+  )
+endif()
+# can delete after this is all worked out
+# https://gitlab.kitware.com/cmake/cmake/-/issues/25702
+find_package(OpenSSL)
+if(NOT DEFINED OPENSSL_FOUND OR NOT ${OPENSSL_FOUND})
+  list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/util")
+  find_package(OpenSSL)
+  if(NOT DEFINED USE_OUR_FIND_OPENSSL OR NOT ${USE_OUR_FIND_OPENSSL})
+    message(FATAL_ERROR "Didn't pickup our FindOpenSSL.cmake")
+  endif()
+  if(NOT DEFINED OPENSSL_FOUND OR NOT ${OPENSSL_FOUND})
+    message(FATAL_ERROR "OpenSSL not found")
+  endif()
+endif()
+message(STATUS "OpenSSL include directories:" ${OPENSSL_INCLUDE_DIR})
 
+include(util/helpers.cmake)
 configure_python_dev_packages()
-find_package(OpenSSL REQUIRED)
+
 find_package(Threads REQUIRED)
 find_package(Boost REQUIRED COMPONENTS system program_options filesystem)
+message(STATUS "Boost include directories:" ${Boost_INCLUDE_DIRS})
 
 # ##############################################################################
 # bootgen
@@ -69,7 +98,6 @@ if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
   set(XAIE_DEBUG "XAIE_DEBUG")
 endif()
 
-target_compile_options(aienginev2 PRIVATE -Wall -Wextra -D__AIECDO__)
 target_compile_definitions(aienginev2 PUBLIC ${XAIE_DEBUG})
 get_target_property(AIERT_SRCS aienginev2 SOURCES)
 list(TRANSFORM AIERT_SRCS PREPEND ${AIERT_SRC_DIR}/)
@@ -135,17 +163,22 @@ if(NOT WIN32)
   target_compile_definitions(xclbinutil-lib
                              PUBLIC ENABLE_JSON_SCHEMA_VALIDATION)
   target_link_libraries(xclbinutil-lib PUBLIC transformcdo)
+else()
+  target_compile_options(xclbinutil-lib PUBLIC "/EHsc")
+  # LINK : fatal error LNK1104: cannot open file
+  # 'libboost_filesystem-vc142-mt-gd-x64-1_74.lib'
+  target_compile_definitions(xclbinutil-lib PUBLIC BOOST_ALL_DYN_LINK)
 endif()
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 target_link_libraries(xclbinutil-lib PUBLIC ${Boost_LIBRARIES} Threads::Threads)
-target_include_directories(xclbinutil-lib PUBLIC ${XRT_BINARY_DIR}/gen)
+target_include_directories(xclbinutil-lib PUBLIC ${XRT_BINARY_DIR}/gen
+                                                 ${Boost_INCLUDE_DIRS})
 set_target_properties(
   xclbinutil-lib
   PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${OUTPUT_DIR}
              ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT_DIR}
-             RUNTIME_OUTPUT_DIRECTORY ${OUTPUT_DIR}
-             MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
+             RUNTIME_OUTPUT_DIRECTORY ${OUTPUT_DIR})
 
 pybind11_add_module(_xclbinutil xaiepy/xclbinutil.cpp)
 target_include_directories(_xclbinutil PRIVATE ${XCLBINUTIL_SOURCE_DIR})

diff --git a/README.md b/README.md
@@ -19,4 +19,19 @@ opcode='XAIE_IO_WRITE'
 reg_off=000000000021d040
 val=0
 mask=000000000fffc000
-```
+```
+
+# Gotchas
+
+If **ON WINDOWS** you're having trouble with 
+
+```shell
+CMake Error at C:/Users/maksim/miniconda3/envs/xaiepy/Lib/site-packages/cmake/data/share/cmake-3.29/Modules/FindPackageHandleStandardArgs.cmake:230 (message):
+  Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the
+  system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY) (found
+  version "3.3.0")
+```
+
+you probably have the win32 developer shell open; open the `x64 Native Tools Command Prompt for VS` instead ([or copy the settings from the win32 powershell to a new link](https://developercommunity.visualstudio.com/t/the-developer-powershell-for-vs-2022-should-use-x6/1568773#T-N10609425)) 🤦.
+
+**Note**: `setup.py` expects OpenSSL to be installed @ `C:\Program Files\OpenSSL`.