Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use vcpkg to build macOS packages #465

Merged
merged 23 commits into from
Dec 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
42cd284
Test macOS build
BewareMyPower Dec 16, 2024
bcfec3f
Build macOS packages via vcpkg
BewareMyPower Dec 16, 2024
14c0869
Add tests for static libraries
BewareMyPower Dec 16, 2024
7c3f5f5
Fetch vcpkg submodules
BewareMyPower Dec 16, 2024
80621f0
Install the libraries
BewareMyPower Dec 16, 2024
092c3b2
Specify OSX architecture
BewareMyPower Dec 16, 2024
de2f80e
Upload artifacts before verifications
BewareMyPower Dec 16, 2024
812bc98
Fix the PULSAR_DIR not set issue
BewareMyPower Dec 16, 2024
b92cf38
Set VCPKG_OSX_DEPLOYMENT_TARGET with 10.15 in triplets
BewareMyPower Dec 16, 2024
b63fb20
Upgrade VCPKG_OSX_DEPLOYMENT_TARGET to 13.7
BewareMyPower Dec 16, 2024
e0b2ecf
Speed up dependencies build by only building release libraries
BewareMyPower Dec 16, 2024
af8fd74
Remove VCPKG_OSX_DEPLOYMENT_TARGET
BewareMyPower Dec 16, 2024
c4c1215
Revert "Remove VCPKG_OSX_DEPLOYMENT_TARGET"
BewareMyPower Dec 16, 2024
796759e
Set minimum osx version to 13.0 and link CoreServices
BewareMyPower Dec 16, 2024
7908cc3
Set CMAKE_OSX_DEPLOYMENT_TARGET for pulsar
BewareMyPower Dec 17, 2024
382c9d9
Apply the patch for vcpkg to disable IPv6 for curl's macOS build
BewareMyPower Dec 17, 2024
9ed491c
Switch to bewaremypower/vcpkg to build curl without frameworks linked
BewareMyPower Dec 17, 2024
0cfe3ae
Restore the deleted workflows
BewareMyPower Dec 17, 2024
8e3eb8d
Update the commit id
BewareMyPower Dec 17, 2024
92bb395
Only use forked vcpkg for macOS build
BewareMyPower Dec 17, 2024
d29abe0
Apply the patch instead of using a forked vcpkg repo
BewareMyPower Dec 27, 2024
8833d1f
Remove the workflow with brew dependencies
BewareMyPower Dec 27, 2024
b5a2a0d
Fetch the submodule in CI
BewareMyPower Dec 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci-build-binary-artifacts.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ jobs:
steps:
- name: checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive

- name: Install dependencies
run: |
Expand Down
42 changes: 4 additions & 38 deletions .github/workflows/ci-pr-validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -311,54 +311,20 @@ jobs:
run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest

cpp-build-macos:
timeout-minutes: 120
name: Build CPP Client on macOS
needs: formatting-check
runs-on: macos-14
steps:
- name: checkout
uses: actions/checkout@v3

- name: Install dependencies
run: brew install openssl protobuf boost zstd snappy googletest

- name: Configure (default)
shell: bash
run: cmake -B ./build-macos -S .

- name: Compile
shell: bash
run: |
cmake --build ./build-macos --parallel --config Release

- name: Build with C++20
shell: bash
run: |
cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20
cmake --build build-macos-cpp20 -j8

cpp-build-macos-static:
timeout-minutes: 120
name: Build CPP Client on macOS with static dependencies
runs-on: macos-14
needs: unit-tests
needs: formatting-check
steps:
- name: checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive

- name: Build libraries
run: ./pkg/mac/build-static-library.sh

- name: Test static libraries
run: |
export PULSAR_DIR=$PWD/pkg/mac/.install
echo "Build with static library"
clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a
./static.out
echo "Build with dynamic library"
clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar
./dynamic.out

# Job that will be required to complete and depends on all the other jobs
check-completion:
name: Check Completion
Expand Down
12 changes: 9 additions & 3 deletions build-support/merge_archives_vcpkg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ if [[ $# -lt 1 ]]; then
fi

CMAKE_BUILD_DIRECTORY=$1
./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
$CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
$(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug)
if [[ $VCPKG_TRIPLET ]]; then
./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
$CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
$(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed/$VCPKG_TRIPLET" -name "*.a" | grep -v debug)
else
./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
$CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
$(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug)
fi
219 changes: 48 additions & 171 deletions pkg/mac/build-static-library.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,182 +18,59 @@
# under the License.
#

set -ex
cd `dirname $0`
set -e
cd `dirname $0`/../..

python3 -m venv venv
source venv/bin/activate
python3 -m pip install pyyaml

MACOSX_DEPLOYMENT_TARGET=10.15
if [[ -z ${ARCH} ]]; then
ARCH=`uname -m`
fi

BUILD_DIR=$PWD/.build
INSTALL_DIR=$PWD/.install
PREFIX=$BUILD_DIR/install
mkdir -p $BUILD_DIR
cp -f ../../build-support/dep-version.py $BUILD_DIR/
cp -f ../../dependencies.yaml $BUILD_DIR/

pushd $BUILD_DIR

BOOST_VERSION=$(./dep-version.py boost)
ZLIB_VERSION=$(./dep-version.py zlib)
OPENSSL_VERSION=$(./dep-version.py openssl)
PROTOBUF_VERSION=$(./dep-version.py protobuf)
ZSTD_VERSION=$(./dep-version.py zstd)
SNAPPY_VERSION=$(./dep-version.py snappy)
CURL_VERSION=$(./dep-version.py curl)

if [ ! -f boost/.done ]; then
echo "Building Boost $BOOST_VERSION"
curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz
tar zxf boost-${BOOST_VERSION}.tar.gz
mkdir -p $PREFIX/include
pushd boost-${BOOST_VERSION}
./bootstrap.sh
./b2 headers
cp -rf boost $PREFIX/include/
popd
mkdir -p boost
touch boost/.done
else
echo "Using cached Boost $BOOST_VERSION"
if [[ -z $ARCH ]]; then
ARCH=$(uname -m)
echo "Use default ARCH: $ARCH"
fi

if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
echo "Building ZLib $ZLIB_VERSION"
curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz
tar zxf zlib-${ZLIB_VERSION}.tar.gz
pushd zlib-$ZLIB_VERSION
CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
make -j16
make install
touch .done
popd
if [[ $ARCH == "x86_64" ]]; then
export VCPKG_TRIPLET=x64-osx
elif [[ $ARCH == "arm64" ]]; then
export VCPKG_TRIPLET=arm64-osx
else
echo "Using cached ZLib $ZLIB_VERSION"
echo "Invalid ARCH: $ARCH"
exit 1
fi

OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then
echo "Building OpenSSL $OPENSSL_VERSION"
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz
tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz

pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
if [[ $ARCH = 'arm64' ]]; then
PLATFORM=darwin64-arm64-cc
else
PLATFORM=darwin64-x86_64-cc
fi
CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM
make -j8 >/dev/null
make install_sw >/dev/null
popd

touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done
else
echo "Using cached OpenSSL $OPENSSL_VERSION"
fi

if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
echo "Building Protobuf $PROTOBUF_VERSION"
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
pushd protobuf-${PROTOBUF_VERSION}
pushd cmake/
# Build protoc that can run on both x86 and arm architectures
cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
-Dprotobuf_BUILD_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=$PREFIX
cmake --build build -j16 --target install
popd

# Retain the library for one architecture so that `ar` can work on the library
pushd $PREFIX/lib
mv libprotobuf.a libprotobuf_universal.a
lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a
popd
touch .done
popd
else
echo "Using cached Protobuf $PROTOBUF_VERSION"
fi

if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
echo "Building ZStd $ZSTD_VERSION"
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
tar zxf zstd-${ZSTD_VERSION}.tar.gz
pushd zstd-${ZSTD_VERSION}
CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \
make -j16 -C lib install-static install-includes
touch .done
popd
else
echo "Using cached ZStd $ZSTD_VERSION"
fi

if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
echo "Building Snappy $SNAPPY_VERSION"
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz
tar zxf ${SNAPPY_VERSION}.tar.gz
pushd snappy-${SNAPPY_VERSION}
# Without this patch, snappy 1.10 will report a sign-compare error, which cannot be suppressed with the -Wno-sign-compare option in CI
curl -O -L https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch
patch <no-werror.patch
CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF
make -j16
make install
touch .done
popd
else
echo "Using cached Snappy $SNAPPY_VERSION"
fi

if [ ! -f curl-${CURL_VERSION}/.done ]; then
echo "Building LibCurl $CURL_VERSION"
CURL_VERSION_=${CURL_VERSION//./_}
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
tar zxf curl-${CURL_VERSION}.tar.gz
pushd curl-${CURL_VERSION}
# Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl.
cp -rf $PREFIX/include/openssl include/
CFLAGS="-I$PREFIX/include -fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
./configure --with-ssl=$PREFIX \
--without-nghttp2 \
--without-libidn2 \
--disable-ldap \
--without-brotli \
--without-secure-transport \
--without-librtmp \
--disable-ipv6 \
--without-libpsl \
--host=$ARCH-apple-darwin \
--prefix=$PREFIX
make -j16 install
touch .done
popd
else
echo "Using cached LibCurl $CURL_VERSION"
fi

popd # pkg/mac
cd ../../ # project root

cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
-DLINK_STATIC=ON \
# Apply the patch to support building libcurl with IPv6 disabled
COMMIT_ID=$(grep "builtin-baseline" vcpkg.json | sed 's/"//g' | sed 's/,//' | awk '{print $2}')
cd vcpkg
git reset --hard $COMMIT_ID
git apply ../pkg/mac/vcpkg-curl-patch.diff
git add ports/curl
git commit -m "Disable IPv6 for macOS in curl"
./bootstrap-vcpkg.sh
./vcpkg x-add-version --all
git add versions/
git commit -m "Update version"
COMMIT_ID=$(git log --pretty=oneline | head -n 1 | awk '{print $1}')
cd ..
sed -i.bak "s/.*builtin-baseline.*/ \"builtin-baseline\": \"$COMMIT_ID\",/" vcpkg.json
sed -i.bak "s/\"version>=\": \"8\.4\.0\"/\"version>=\": \"8.4.0#1\"/" vcpkg.json

INSTALL_DIR=$PWD/pkg/mac/.install
set -x
cmake -B build-osx \
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \
-DINTEGRATE_VCPKG=ON \
-DVCPKG_OVERLAY_TRIPLETS=$PWD/vcpkg-triplets \
-DVCPKG_TARGET_TRIPLET=$VCPKG_TRIPLET \
-DCMAKE_OSX_ARCHITECTURES=$ARCH \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTS=OFF \
-DBUILD_PERF_TOOLS=OFF \
-DBUILD_DYNAMIC_LIB=ON \
-DBUILD_STATIC_LIB=ON \
-DCMAKE_OSX_ARCHITECTURES=${ARCH} \
-DCMAKE_PREFIX_PATH=$PREFIX \
-DOPENSSL_ROOT_DIR=$PREFIX \
-DPROTOC_PATH=$PREFIX/bin/protoc \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-DCMAKE_BUILD_TYPE=Release
cmake --build build-static -j16 --target install
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
cmake --build build-osx -j16 --target install

./build-support/merge_archives_vcpkg.sh $PWD/build-osx
cp ./build-osx/libpulsarwithdeps.a $INSTALL_DIR/lib/

# Test the libraries
clang++ win-examples/example.cc -o dynamic.out -std=c++11 -arch $ARCH -I $INSTALL_DIR/include -L $INSTALL_DIR/lib -Wl,-rpath $INSTALL_DIR/lib -lpulsar
./dynamic.out
clang++ win-examples/example.cc -o static.out -std=c++11 -arch $ARCH -I $INSTALL_DIR/include $INSTALL_DIR/lib/libpulsarwithdeps.a
./static.out
27 changes: 27 additions & 0 deletions pkg/mac/vcpkg-curl-patch.diff
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
index bdc544e9e..340d93865 100644
--- a/ports/curl/portfile.cmake
+++ b/ports/curl/portfile.cmake
@@ -64,6 +64,10 @@ if(VCPKG_TARGET_IS_WINDOWS)
list(APPEND OPTIONS -DENABLE_UNICODE=ON)
endif()

+if(VCPKG_TARGET_IS_OSX)
+ list(APPEND OPTIONS -DENABLE_IPV6=OFF)
+endif()
+
vcpkg_cmake_configure(
SOURCE_PATH "${SOURCE_PATH}"
OPTIONS
diff --git a/ports/curl/vcpkg.json b/ports/curl/vcpkg.json
index e028d3897..a63858e34 100644
--- a/ports/curl/vcpkg.json
+++ b/ports/curl/vcpkg.json
@@ -1,6 +1,7 @@
{
"name": "curl",
"version": "8.4.0",
+ "port-version": 1,
"description": "A library for transferring data with URLs",
"homepage": "https://curl.se/",
"license": "curl AND ISC AND BSD-3-Clause",
9 changes: 9 additions & 0 deletions vcpkg-triplets/arm64-osx.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
set(VCPKG_TARGET_ARCHITECTURE arm64)
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE static)

set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
set(VCPKG_OSX_ARCHITECTURES arm64)
set(VCPKG_OSX_DEPLOYMENT_TARGET 13.0)

set(VCPKG_BUILD_TYPE release)
9 changes: 9 additions & 0 deletions vcpkg-triplets/x64-osx.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
set(VCPKG_TARGET_ARCHITECTURE x64)
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE static)

set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
set(VCPKG_OSX_ARCHITECTURES x86_64)
set(VCPKG_OSX_DEPLOYMENT_TARGET 13.0)

set(VCPKG_BUILD_TYPE release)
Loading