windows build

confluentinc · Feb 6, 2023 · 53f53ba · 53f53ba
1 parent 6a3f438
commit 53f53ba
Show file tree

Hide file tree

Showing 5 changed files with 130 additions and 11 deletions.
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -12,8 +12,8 @@ global_job_config:
       - checkout
 blocks:
   - name: "Wheels: OSX x64"
-    run:
-      when: "tag =~ '.*'"
+    # run:
+    #   when: "tag =~ '.*'"
     dependencies: []
     task:
       agent:
@@ -31,8 +31,8 @@ blocks:
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
   - name: "Wheels: OSX arm64"
-    run:
-      when: "tag =~ '.*'"
+    # run:
+    #   when: "tag =~ '.*'"
     dependencies: []
     task:
       agent:
@@ -51,7 +51,6 @@ blocks:
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
-
   - name: Source package verification with Python 3 (OSX x64) +docs
     dependencies: []
     task:
@@ -81,8 +80,8 @@ blocks:
             - python setup.py build && python setup.py install
             - make docs
   - name: "Wheels: Linux arm64"
-    run:
-      when: "tag =~ '.*'"
+    # run:
+    #   when: "tag =~ '.*'"
     dependencies: []
     task:
       agent:
@@ -99,3 +98,66 @@ blocks:
             - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
             - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-linux-${ARCH}.tgz
+  - name: "Wheels: Windows"
+    # run:
+    #   when: "tag =~ '.*'"
+    dependencies: []
+    task:
+      agent:
+          machine:
+            type: s1-prod-windows
+      env_vars:
+        - name: OS_NAME
+          value: windows
+        - name: ARCH
+          value: x64
+      prologue:
+        commands:
+          - cache restore msys2-x64
+          - ".\\tools\\mingw-w64\\setup-msys2.ps1"
+          - $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
+          - bash -lc './tools/mingw-w64/msys2-dependencies.sh'
+          - cache delete msys2-x64
+          - cache store msys2-x64 c:/msys64
+      jobs:
+        - name: Build
+          env_vars:
+            - name: CHERE_INVOKING
+              value: 'yes'
+            - name: MSYSTEM
+              value: UCRT64
+          commands:
+            - bash tools/mingw-w64/semaphore_commands.sh
+            - bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest
+            - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
+            - ls
+            - tar -czf wheelhouse-windows-${ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-windows-${ARCH}.tgz
+  - name: Source package verification with Python 3 (Linux x64) +docs
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu20-04-amd64-3
+      env_vars:
+        - name: OS_NAME
+          value: linux
+        - name: ARCH
+          value: x64
+      jobs:
+        - name: Build
+          commands:
+            # use a virtualenv
+            - python3 -m venv _venv && source _venv/bin/activate
+            - pip install -r docs/requirements.txt
+            - pip install -U protobuf
+            # install librdkafka
+            - lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
+            - tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest
+            - export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include"
+            - export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
+            - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
+            - export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir"
+            # install confluent-kafka
+            - python setup.py build && python setup.py install
+            - make docs
diff --git a/tools/mingw-w64/msys2-dependencies.sh b/tools/mingw-w64/msys2-dependencies.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+export msys2='cmd //C RefreshEnv.cmd '
+export msys2+='& set MSYS=winsymlinks:nativestrict '
+export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start'
+export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --"
+export msys2+=" -msys2 -c "\"\$@"\" --"
+
+# Have to update pacman first or choco upgrade will failure due to migration
+# to zstd instead of xz compression
+$msys2 pacman -Sy --noconfirm pacman
+
+## Install more MSYS2 packages from https://packages.msys2.org/base here
+$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc
+
+## Install unzip
+$msys2 pacman --sync --noconfirm --needed unzip
diff --git a/tools/mingw-w64/semaphore_commands.sh b/tools/mingw-w64/semaphore_commands.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+pacman -S python --version 3.8.0
+
+set -e
+
+export PATH="$PATH;C:\Python38;C:\Python38\Scripts"
+export MAKE=mingw32-make  # so that Autotools can find it
+
+cmd /c mklink /D C:\Python38\python3.exe C:\Python38\python.exe
+
+python -m pip install cibuildwheel==2.12.0
diff --git a/tools/mingw-w64/setup-msys2.ps1 b/tools/mingw-w64/setup-msys2.ps1
@@ -0,0 +1,31 @@
+# Install (if necessary) and set up msys2.
+
+
+$url="https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-base-x86_64-20221028.sfx.exe"
+$sha256="e365b79b4b30b6f4baf34bd93f3d2a41c0a92801c7a96d79cddbfca1090a0554"
+
+
+if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) {
+    echo "Downloading and installing msys2 to c:\msys64"
+
+    (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe')
+
+    # Verify checksum
+    (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256
+
+    # Install msys2
+    .\msys2-installer.exe -y -oc:\
+
+    Remove-Item msys2-installer.exe
+
+    # Set up msys2 the first time
+    echo "Setting up msys"
+    c:\msys64\usr\bin\bash -lc ' '
+
+} else {
+    echo "Using previously installed msys2"
+}
+
+# Update packages
+echo "Updating msys2 packages"
+c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'"
diff --git a/tools/wheels/build-wheels.bat b/tools/wheels/build-wheels.bat
@@ -25,12 +25,8 @@ set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% -
 
 set PATH=%PATH%;c:\Program Files\Git\bin\
 
-python -m pip install cibuildwheel==2.12.0 || goto :error
-
 python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows || goto :error
 
-dir %WHEELHOUSE%
-
 goto :eof
 
 :usage