Migarting travis jobs to Semaphore

.semaphore/semaphore.yml

@@ -10,6 +10,7 @@ global_job_config:
   prologue:
     commands:
       - checkout
+      - mkdir artifacts
 blocks:
   - name: "Wheels: OSX x64"
     run:
@@ -29,7 +30,7 @@ blocks:
           commands:
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
-            - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
+            - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
   - name: "Wheels: OSX arm64"
     run:
       when: "tag =~ '.*'"
@@ -50,40 +51,100 @@ blocks:
           commands:
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
-            - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
-
-  - name: Source package verification with Python 3 (OSX x64) +docs
+            - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
+  - name: "Wheels: Linux arm64"
+    run:
+      when: "tag =~ '.*'"
     dependencies: []
     task:
       agent:
         machine:
-          type: s1-prod-macos
+          type: s1-prod-ubuntu20-04-arm64-1
       env_vars:
         - name: OS_NAME
-          value: osx
+          value: linux
         - name: ARCH
           value: arm64
       jobs:
         - name: Build
           commands:
-            # use a virtualenv
-            - python3 -m venv _venv && source _venv/bin/activate
-            - pip install -r docs/requirements.txt
-            - pip install -U protobuf
-            # install librdkafka
-            - lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
-            - tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest
-            - export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include"
-            - export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
-            - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
-            - export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir"
-            # install confluent-kafka
-            - python setup.py build && python setup.py install
-            - make docs
-  - name: "Wheels: Linux arm64"
+            - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
+            - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
+  - name: "Wheels: Linux x64"
+    run:
+      when: "tag =~ '.*'"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu20-04-amd64-3
+      env_vars:
+        - name: OS_NAME
+          value: linux
+        - name: ARCH
+          value: x64
+      jobs:
+        - name: Build
+          commands:
+            - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
+            - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
+  - name: "Wheels: Windows"
     run:
       when: "tag =~ '.*'"
     dependencies: []
+    task:
+      agent:
+          machine:
+            type: s1-prod-windows
+      env_vars:
+        - name: OS_NAME
+          value: windows
+        - name: ARCH
+          value: x64
+      prologue:
+        commands:
+          - cache restore msys2-x64
+          - ".\\tools\\mingw-w64\\setup-msys2.ps1"
+          - $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
+          - bash -lc './tools/mingw-w64/msys2-dependencies.sh'
+          - cache delete msys2-x64
+          - cache store msys2-x64 c:/msys64
+      jobs:
+        - name: Build
+          env_vars:
+            - name: CHERE_INVOKING
+              value: 'yes'
+            - name: MSYSTEM
+              value: UCRT64
+          commands:
+            - bash tools/mingw-w64/semaphore_commands.sh
+            - bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest
+            - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
+            - tar -czf wheelhouse-windows-${Env:ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-windows-${Env:ARCH}.tgz --destination artifacts/wheels-${Env:OS_NAME}-${Env:ARCH}.tgz/
+  - name: "Source package verification and Integration tests with Python 3 (Linux x64)"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu20-04-amd64-2
+      env_vars:
+        - name: OS_NAME
+          value: linux
+        - name: ARCH
+          value: x64
+      jobs:
+        - name: Build
+          commands:
+            - sem-version python 3.8
+            # use a virtualenv
+            - python3 -m venv _venv && source _venv/bin/activate
+            - chmod u+r+x tools/source-package-verification.sh
+            - tools/source-package-verification.sh
+  - name: "Source package verification with Python 3 (Linux arm64)"
+    dependencies: []
     task:
       agent:
         machine:
@@ -96,6 +157,71 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
-            - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
-            - artifact push workflow wheelhouse-linux-${ARCH}.tgz
+            - sem-version python 3.8
+            # use a virtualenv
+            - python3 -m venv _venv && source _venv/bin/activate
+            - chmod u+r+x tools/source-package-verification.sh
+            - tools/source-package-verification.sh
+  - name: "Source package verification with Python 3 (OSX x64) +docs"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-macos
+      env_vars:
+        - name: OS_NAME
+          value: osx
+        - name: ARCH
+          value: x64
+      jobs:
+        - name: Build
+          commands:
+            - sem-version python 3.8
+            # use a virtualenv
+            - python3 -m venv _venv && source _venv/bin/activate
+            - chmod u+r+x tools/source-package-verification.sh
+            - tools/source-package-verification.sh
+  - name: "Source package verification with Python 3 (OSX arm64) +docs"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-macos-arm64
+      env_vars:
+        - name: OS_NAME
+          value: osx
+        - name: ARCH
+          value: arm64
+      jobs:
+        - name: Build
+          commands:
+            - sem-version python 3.8
+            # use a virtualenv
+            - python3 -m venv _venv && source _venv/bin/activate
+            - chmod u+r+x tools/source-package-verification.sh
+            - tools/source-package-verification.sh
+  - name: "Packaging"
+    run:
+      when: "tag =~ '.*'"
+    dependencies: 
+      - "Wheels: OSX x64"
+      - "Wheels: OSX arm64"
+      - "Wheels: Linux arm64"
+      - "Wheels: Linux x64"
+      - "Wheels: Windows"
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu20-04-amd64-3
+      jobs:
+        - name: "Packaging all artifacts"
+          commands:
+            - artifact pull workflow artifacts
+            - cd artifacts
+            - ls *.tgz |xargs -n1 tar -xvf
+            - tar cvf confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz wheelhouse/
+            - ls -la
+            - sha256sum confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz
+            - cd ..
+            - artifact push project artifacts/confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz --destination confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz
+            - echo Thank you

CHANGELOG.md

@@ -8,6 +8,8 @@
 - Added support for Default num_partitions in CreateTopics Admin API.
 - Added support for password protected private key in CachedSchemaRegistryClient.
 - Add reference support in Schema Registry client. (@RickTalken, #1304)
+- Migrated travis jobs to Semaphore CI (#1503)
+
 
 ## v2.0.2
 

tools/mingw-w64/msys2-dependencies.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+export msys2='cmd //C RefreshEnv.cmd '
+export msys2+='& set MSYS=winsymlinks:nativestrict '
+export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start'
+export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --"
+export msys2+=" -msys2 -c "\"\$@"\" --"
+
+# Have to update pacman first or choco upgrade will failure due to migration
+# to zstd instead of xz compression
+$msys2 pacman -Sy --noconfirm pacman
+
+## Install more MSYS2 packages from https://packages.msys2.org/base here
+$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc
+
+## Install unzip
+$msys2 pacman --sync --noconfirm --needed unzip

tools/mingw-w64/semaphore_commands.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+$msys2 pacman -S python --version 3.8.0
+
+set -e
+
+export PATH="$PATH;C:\Python38;C:\Python38\Scripts"
+export MAKE=mingw32-make  # so that Autotools can find it
+
+cmd /c mklink /D C:\Python38\python3.exe C:\Python38\python.exe
+
+python -m pip install cibuildwheel==2.12.0

tools/mingw-w64/setup-msys2.ps1

@@ -0,0 +1,31 @@
+# Install (if necessary) and set up msys2.
+
+
+$url="https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-base-x86_64-20221028.sfx.exe"
+$sha256="e365b79b4b30b6f4baf34bd93f3d2a41c0a92801c7a96d79cddbfca1090a0554"
+
+
+if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) {
+    echo "Downloading and installing msys2 to c:\msys64"
+
+    (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe')
+
+    # Verify checksum
+    (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256
+
+    # Install msys2
+    .\msys2-installer.exe -y -oc:\
+
+    Remove-Item msys2-installer.exe
+
+    # Set up msys2 the first time
+    echo "Setting up msys"
+    c:\msys64\usr\bin\bash -lc ' '
+
+} else {
+    echo "Using previously installed msys2"
+}
+
+# Update packages
+echo "Updating msys2 packages"
+c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'"