From 94499e6b2a8664c200317a528da5326c4846259e Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 4 Dec 2022 13:31:46 -0800 Subject: [PATCH 1/7] fix: install arm brew on MacOS Arm to install libsodium --- .github/workflows/CI.yml | 11 +++++++++-- script/macos-arm-deps.sh | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 script/macos-arm-deps.sh diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a20fa3f9..9231871a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -106,11 +106,18 @@ jobs: architecture: ${{ matrix.node_arch }} cache: "pnpm" - - name: Install Mac-OS Dependencies - if: contains(matrix.os, 'macos') + - name: Install Mac-OS ${{ matrix.ARCH }} Dependencies + if: ${{ contains(matrix.os, 'macos') && matrix.ARCH == 'x86_64' }} run: | brew install libsodium gnutls + - name: Install Mac-OS ${{ matrix.ARCH }} Dependencies + if: ${{ contains(matrix.os, 'macos') && matrix.ARCH == 'arm64' }} + run: | + brew uninstall libsodium --force --ignore-dependencies + chmod +x ./script/macos-arm-deps.sh + ./script/macos-arm-deps.sh + - name: Install Dependencies and Build if: ${{ !matrix.docker }} run: pnpm install diff --git a/script/macos-arm-deps.sh b/script/macos-arm-deps.sh new file mode 100644 index 00000000..59639816 --- /dev/null +++ b/script/macos-arm-deps.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +#! Install arm-brew on x86 MacOS Arm +#! Based on https://github.com/Homebrew/discussions/discussions/2843#discussioncomment-2243610 + +bottle_tag="arm64_big_sur" # Macos 11 is big sure +dependencies="libsodium gnutls" + +mkdir -p ~/arm-target/bin +mkdir -p ~/arm-target/brew-cache +export PATH="$HOME/arm-target/bin:$PATH" + +PREV_PWD="$PWD" +cd ~/arm-target + +mkdir arm-homebrew +curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm-homebrew +ln -s ~/arm-target/arm-homebrew/bin/brew ~/arm-target/bin/arm-brew + +export HOMEBREW_CACHE=~/arm-target/brew-cache +export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + +arm-brew fetch --deps --bottle-tag=$bottle_tag $dependencies | + grep -E "(Downloaded to:|Already downloaded:)" | + grep -v pkg-config | + awk '{ print $3 }' | + xargs -n 1 arm-brew install --ignore-dependencies --force-bottle + +# Install host version of pkg-config so we can call it in the build system +arm-brew install pkg-config +ln -s ~/arm-target/arm-homebrew/bin/pkg-config ~/arm-target/bin/arm-pkg-config + +cd "$PREV_PWD" From 369804a882f7f3f29459b550c305514148258fdb Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 4 Dec 2022 15:51:32 -0800 Subject: [PATCH 2/7] fix: add the arm libsodium to path --- .github/workflows/CI.yml | 6 +++--- script/macos-arm-deps.sh | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9231871a..094a26e4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -106,17 +106,17 @@ jobs: architecture: ${{ matrix.node_arch }} cache: "pnpm" - - name: Install Mac-OS ${{ matrix.ARCH }} Dependencies + - name: Install Mac-OS x86_64 Dependencies if: ${{ contains(matrix.os, 'macos') && matrix.ARCH == 'x86_64' }} run: | brew install libsodium gnutls - - name: Install Mac-OS ${{ matrix.ARCH }} Dependencies + - name: Install Mac-OS arm64 Dependencies if: ${{ contains(matrix.os, 'macos') && matrix.ARCH == 'arm64' }} run: | brew uninstall libsodium --force --ignore-dependencies chmod +x ./script/macos-arm-deps.sh - ./script/macos-arm-deps.sh + source ./script/macos-arm-deps.sh - name: Install Dependencies and Build if: ${{ !matrix.docker }} diff --git a/script/macos-arm-deps.sh b/script/macos-arm-deps.sh index 59639816..44532667 100644 --- a/script/macos-arm-deps.sh +++ b/script/macos-arm-deps.sh @@ -25,10 +25,27 @@ arm-brew fetch --deps --bottle-tag=$bottle_tag $dependencies | grep -E "(Downloaded to:|Already downloaded:)" | grep -v pkg-config | awk '{ print $3 }' | - xargs -n 1 arm-brew install --ignore-dependencies --force-bottle + xargs -n 1 arm-brew install --force-bottle # Install host version of pkg-config so we can call it in the build system arm-brew install pkg-config ln -s ~/arm-target/arm-homebrew/bin/pkg-config ~/arm-target/bin/arm-pkg-config +export PATH="$HOME/arm-target/arm-homebrew/bin/:$PATH" +export PATH="$HOME/arm-target/arm-homebrew/lib/:$PATH" + cd "$PREV_PWD" + +# libsodium +export PATH="$HOME/arm-target/Cellar/libsodium/1.0.18_1/lib:$PATH" +export PKG_CONFIG_PATH="$HOME/arm-target/Cellar/libsodium/1.0.18_1/lib:$PKG_CONFIG_PATH" + +# gnutils +export GUILE_TLS_CERTIFICATE_DIRECTORY=/Users/runner/arm-target/etc/gnutls/ +export PATH="$HOME/arm-target/opt/gnu-sed/libexec/gnubin:$PATH" +export PATH="$HOME/arm-target/opt/m4/bin:$PATH" +export LDFLAGS="-L$HOME/arm-target/opt/readline/lib $LDFLAGS" +export CPPFLAGS="-I$HOME/arm-target/opt/readline/include $CPPFLAGS" +export PKG_CONFIG_PATH="$HOME/arm-target/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH" +export PKG_CONFIG_PATH="$HOME/arm-target/opt/openssl@1.1/lib/pkgconfig:$PKG_CONFIG_PATH" +export PKG_CONFIG_PATH="$HOME/arm-target/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH" From 4eadc1871b26fb865aa2e5806be99d145533e95c Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 4 Dec 2022 16:57:08 -0800 Subject: [PATCH 3/7] fix: do not install gnutls for macos arm --- script/macos-arm-deps.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/script/macos-arm-deps.sh b/script/macos-arm-deps.sh index 44532667..3b0e38fa 100644 --- a/script/macos-arm-deps.sh +++ b/script/macos-arm-deps.sh @@ -5,7 +5,7 @@ set -e #! Based on https://github.com/Homebrew/discussions/discussions/2843#discussioncomment-2243610 bottle_tag="arm64_big_sur" # Macos 11 is big sure -dependencies="libsodium gnutls" +dependencies="libsodium" mkdir -p ~/arm-target/bin mkdir -p ~/arm-target/brew-cache @@ -39,13 +39,3 @@ cd "$PREV_PWD" # libsodium export PATH="$HOME/arm-target/Cellar/libsodium/1.0.18_1/lib:$PATH" export PKG_CONFIG_PATH="$HOME/arm-target/Cellar/libsodium/1.0.18_1/lib:$PKG_CONFIG_PATH" - -# gnutils -export GUILE_TLS_CERTIFICATE_DIRECTORY=/Users/runner/arm-target/etc/gnutls/ -export PATH="$HOME/arm-target/opt/gnu-sed/libexec/gnubin:$PATH" -export PATH="$HOME/arm-target/opt/m4/bin:$PATH" -export LDFLAGS="-L$HOME/arm-target/opt/readline/lib $LDFLAGS" -export CPPFLAGS="-I$HOME/arm-target/opt/readline/include $CPPFLAGS" -export PKG_CONFIG_PATH="$HOME/arm-target/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH" -export PKG_CONFIG_PATH="$HOME/arm-target/opt/openssl@1.1/lib/pkgconfig:$PKG_CONFIG_PATH" -export PKG_CONFIG_PATH="$HOME/arm-target/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH" From 6cccfcb469c041804aca93e5a857e3e700af54f0 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 4 Dec 2022 18:41:45 -0800 Subject: [PATCH 4/7] fix: use arm-pkg-config to get the path to libsodium --- script/macos-arm-deps.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/script/macos-arm-deps.sh b/script/macos-arm-deps.sh index 3b0e38fa..a196c987 100644 --- a/script/macos-arm-deps.sh +++ b/script/macos-arm-deps.sh @@ -11,16 +11,21 @@ mkdir -p ~/arm-target/bin mkdir -p ~/arm-target/brew-cache export PATH="$HOME/arm-target/bin:$PATH" +# Download Homebrew under ~/arm-target PREV_PWD="$PWD" cd ~/arm-target - mkdir arm-homebrew curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm-homebrew +cd "$PREV_PWD" + +# Add arm-brew binary ln -s ~/arm-target/arm-homebrew/bin/brew ~/arm-target/bin/arm-brew +# Homebrew env variables export HOMEBREW_CACHE=~/arm-target/brew-cache export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# Install the given dependencies for the given bottle_tag arm-brew fetch --deps --bottle-tag=$bottle_tag $dependencies | grep -E "(Downloaded to:|Already downloaded:)" | grep -v pkg-config | @@ -29,13 +34,12 @@ arm-brew fetch --deps --bottle-tag=$bottle_tag $dependencies | # Install host version of pkg-config so we can call it in the build system arm-brew install pkg-config -ln -s ~/arm-target/arm-homebrew/bin/pkg-config ~/arm-target/bin/arm-pkg-config +# Add the installed binaries/libraries to the path export PATH="$HOME/arm-target/arm-homebrew/bin/:$PATH" export PATH="$HOME/arm-target/arm-homebrew/lib/:$PATH" -cd "$PREV_PWD" - # libsodium -export PATH="$HOME/arm-target/Cellar/libsodium/1.0.18_1/lib:$PATH" -export PKG_CONFIG_PATH="$HOME/arm-target/Cellar/libsodium/1.0.18_1/lib:$PKG_CONFIG_PATH" +SODIUM_PATH=$(~/arm-target/arm-homebrew/bin/pkg-config libsodium --libs) +export PATH="$SODIUM_PATH:$PATH" +export PKG_CONFIG_PATH="$SODIUM_PATH:$PKG_CONFIG_PATH" From 188cc3a6f01060665f9b84480fe974f5b547352e Mon Sep 17 00:00:00 2001 From: Bartel Eerdekens Date: Mon, 5 Dec 2022 11:13:23 +0100 Subject: [PATCH 5/7] fix: make macos-arm-deps.sh executable + fix paths + fix re-running --- .github/workflows/CI.yml | 1 - script/macos-arm-deps.sh | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) mode change 100644 => 100755 script/macos-arm-deps.sh diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 094a26e4..ffb5d8de 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -115,7 +115,6 @@ jobs: if: ${{ contains(matrix.os, 'macos') && matrix.ARCH == 'arm64' }} run: | brew uninstall libsodium --force --ignore-dependencies - chmod +x ./script/macos-arm-deps.sh source ./script/macos-arm-deps.sh - name: Install Dependencies and Build diff --git a/script/macos-arm-deps.sh b/script/macos-arm-deps.sh old mode 100644 new mode 100755 index a196c987..fed373d8 --- a/script/macos-arm-deps.sh +++ b/script/macos-arm-deps.sh @@ -14,12 +14,12 @@ export PATH="$HOME/arm-target/bin:$PATH" # Download Homebrew under ~/arm-target PREV_PWD="$PWD" cd ~/arm-target -mkdir arm-homebrew +mkdir -p arm-homebrew curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm-homebrew cd "$PREV_PWD" # Add arm-brew binary -ln -s ~/arm-target/arm-homebrew/bin/brew ~/arm-target/bin/arm-brew +ln -sf ~/arm-target/arm-homebrew/bin/brew ~/arm-target/bin/arm-brew # Homebrew env variables export HOMEBREW_CACHE=~/arm-target/brew-cache @@ -28,18 +28,19 @@ export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 # Install the given dependencies for the given bottle_tag arm-brew fetch --deps --bottle-tag=$bottle_tag $dependencies | grep -E "(Downloaded to:|Already downloaded:)" | + grep -E ".tar.gz" | grep -v pkg-config | awk '{ print $3 }' | - xargs -n 1 arm-brew install --force-bottle + xargs -n 1 arm-brew reinstall --force-bottle # Install host version of pkg-config so we can call it in the build system -arm-brew install pkg-config +arm-brew reinstall pkg-config # Add the installed binaries/libraries to the path -export PATH="$HOME/arm-target/arm-homebrew/bin/:$PATH" -export PATH="$HOME/arm-target/arm-homebrew/lib/:$PATH" +export PATH="$HOME/arm-target/bin/:$PATH" +export PATH="$HOME/arm-target/lib/:$PATH" # libsodium -SODIUM_PATH=$(~/arm-target/arm-homebrew/bin/pkg-config libsodium --libs) +SODIUM_PATH=$(~/arm-target/bin/pkg-config libsodium --libs-only-L | sed -e 's/-L//g') # print only -L and replace "-L" itself export PATH="$SODIUM_PATH:$PATH" export PKG_CONFIG_PATH="$SODIUM_PATH:$PKG_CONFIG_PATH" From af8f5c424d6d8169d098daca74f5437174ed9f46 Mon Sep 17 00:00:00 2001 From: Bartel Eerdekens Date: Mon, 5 Dec 2022 13:26:12 +0100 Subject: [PATCH 6/7] fix: faster re-install, ignore warning when already installed + fix pnpm install for arm64 --- script/macos-arm-deps.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/macos-arm-deps.sh b/script/macos-arm-deps.sh index fed373d8..a62758e5 100755 --- a/script/macos-arm-deps.sh +++ b/script/macos-arm-deps.sh @@ -31,10 +31,10 @@ arm-brew fetch --deps --bottle-tag=$bottle_tag $dependencies | grep -E ".tar.gz" | grep -v pkg-config | awk '{ print $3 }' | - xargs -n 1 arm-brew reinstall --force-bottle + xargs -n 1 arm-brew install --force-bottle || true # Install host version of pkg-config so we can call it in the build system -arm-brew reinstall pkg-config +arm-brew install pkg-config || true # Add the installed binaries/libraries to the path export PATH="$HOME/arm-target/bin/:$PATH" @@ -44,3 +44,4 @@ export PATH="$HOME/arm-target/lib/:$PATH" SODIUM_PATH=$(~/arm-target/bin/pkg-config libsodium --libs-only-L | sed -e 's/-L//g') # print only -L and replace "-L" itself export PATH="$SODIUM_PATH:$PATH" export PKG_CONFIG_PATH="$SODIUM_PATH:$PKG_CONFIG_PATH" +export npm_config_target_arch=arm64 From 1cfccdb231d267d5999401135f43565d488c8bf3 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 5 Dec 2022 11:37:33 -0800 Subject: [PATCH 7/7] ci: make the macos arm variables permanent --- .github/workflows/CI.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ffb5d8de..727c4e92 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -116,6 +116,9 @@ jobs: run: | brew uninstall libsodium --force --ignore-dependencies source ./script/macos-arm-deps.sh + echo "PATH=$PATH" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> $GITHUB_ENV + echo "npm_config_target_arch=$npm_config_target_arch" >> $GITHUB_ENV - name: Install Dependencies and Build if: ${{ !matrix.docker }}