ci: Build statically linked bpf-linker in CI

Build LLVM and bpf-linker with statically linked libLLVM for multiple architectures. Include the resulting build artifacts on the release pages, so they can be installed with `cargo binstall`
aya-rs · Sep 21, 2024 · 1439a52 · 1439a52
1 parent 07365b6
commit 1439a52
Show file tree

Hide file tree

Showing 5 changed files with 178 additions and 71 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -48,21 +48,58 @@ jobs:
  run: cargo fmt --all -- --check
 
  build:
- runs-on: ubuntu-22.04
+ runs-on: ${{ matrix.target.os }}
  strategy:
  fail-fast: false
  matrix:
  rust:
  - stable
  - beta
  - nightly
- llvm:
- - 19
- - source
+ target:
+ - os: macos-13
+ target: aarch64-apple-darwin
+ target-llvm: aarch64-apple-darwin
+ - os: macos-13
+ target: x86_64-apple-darwin
+ target-llvm: x86_64-apple-darwin
+ # We use the GNU builds of LLVM both for GNU and musl builds of
+ # bpf-linker - it doesn't affect the type of libc being picked by
+ # Cargo and musl Rust toolchains come with their own copies of
+ # libc.a.
+ - os: ubuntu-22.04
+ packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+ target: aarch64-unknown-linux-gnu
+ target-llvm: aarch64-linux-gnu
+ - os: ubuntu-22.04
+ packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+ target: aarch64-unknown-linux-musl
+ target-llvm: aarch64-linux-gnu
+ - os: ubuntu-22.04
+ packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross
+ target: riscv64gc-unknown-linux-gnu
+ target-llvm: riscv64-linux-gnu
+ - os: ubuntu-22.04
+ packages:
+ target: x86_64-unknown-linux-gnu
+ target-llvm: x86_64-linux-gnu
+ - os: ubuntu-22.04
+ packages:
+ target: x86_64-unknown-linux-musl
+ target-llvm: x86_64-linux-gnu
  name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }}
  needs: llvm
 
  env:
+ CARGO_BUILD_TARGET: ${{ matrix.target.target }}
+ # We use the GNU sysroot as an LD path both for GNU and musl builds of
+ # bpf-linker - the user-space emulator and the path are used only for
+ # executing `llvm-config` and test binaries.
+ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+ CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/riscv64-linux-gnu
+ CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/riscv64-linux-gnu
+ LLVM_SYS_CROSS_LD_PREFIX: /usr/${{ matrix.target.target-llvm }}
  RUST_BACKTRACE: full
 
  steps:
@@ -80,8 +117,7 @@ jobs:
  with:
  toolchain: ${{ matrix.rust }}
  components: rust-src
- # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15.
- targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl
+ targets: ${{ matrix.target.target }}
 
  - uses: Swatinem/rust-cache@v2
 
@@ -92,7 +128,7 @@ jobs:
  run: cargo build
 
  - name: Install dependencies
- if: matrix.rust == 'nightly'
+ if: runner.os == 'Linux'
  # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
  # enum values which was added in clang 15[1].
  #
@@ -108,42 +144,19 @@ jobs:
  wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
  echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
  sudo apt update
- sudo apt -y install clang gcc-multilib
-
- - name: Install LLVM
- if: matrix.llvm != 'source'
- run: |
- set -euxo pipefail
- wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
- echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list
-
- sudo apt update
- # TODO(vadorovsky): Remove the requirement of libpolly.
- #
- # Packages from apt.llvm.org are being built all at once, with one
- # cmake build with superset of options, then different binaries and
- # libraries are being included in different packages.
- #
- # That results in `llvm-config --libname --link-static` mentioning
- # libpolly, even if it's not installed. The output of that command is
- # being used in build.rs of llvm-sys, so building llvm-sys on such
- # system is complaining about lack of libpolly.
- #
- # Hopefully that nightmare goes away once we switch to binstalls and
- # ditch the system LLVM option.
- sudo apt -y install llvm-${{ matrix.llvm }}-dev libpolly-${{ matrix.llvm }}-dev
- echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH
+ sudo apt -y install \
+ clang \
+ gcc-multilib \
+ ${{ matrix.target.packages }}
 
  - name: Restore LLVM
- if: matrix.llvm == 'source'
  uses: actions/cache/restore@v4
  with:
  path: llvm-install
- key: ${{ needs.llvm.outputs.cache-key }}
+ key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target.target-llvm)] }}
  fail-on-cache-miss: true
 
  - name: Add LLVM to PATH && LD_LIBRARY_PATH
- if: matrix.llvm == 'source'
  run: |
  set -euxo pipefail
  echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH

diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml
@@ -3,13 +3,47 @@ name: LLVM
 on:
  workflow_call:
  outputs:
- cache-key:
- value: ${{ jobs.llvm.outputs.cache-key }}
+ cache-key-aarch64-apple-darwin:
+ value: ${{ jobs.llvm.outputs.cache-key-aarch64-apple-darwin }}
+ cache-key-x86_64-apple-darwin:
+ value: ${{ jobs.llvm.outputs.cache-key-x86_64-apple-darwin }}
+ cache-key-aarch64-linux-gnu:
+ value: ${{ jobs.llvm.outputs.cache-key-aarch64-linux-gnu }}
+ cache-key-riscv64-linux-gnu:
+ value: ${{ jobs.llvm.outputs.cache-key-riscv64-linux-gnu }}
+ cache-key-x86_64-linux-gnu:
+ value: ${{ jobs.llvm.outputs.cache-key-x86_64-linux-gnu }}
 
 jobs:
  llvm:
- runs-on: ubuntu-22.04
- name: llvm
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ include:
+ - os: macos-13
+ processor: arm64
+ system: Darwin
+ target: aarch64-apple-darwin
+ - os: macos-13
+ processor: x86_64
+ system: Darwin
+ target: x86_64-apple-darwin
+ - os: ubuntu-22.04
+ packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+ processor: aarch64
+ system: Linux
+ target: aarch64-linux-gnu
+ - os: ubuntu-22.04
+ packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross
+ processor: riscv64
+ system: Linux
+ target: riscv64-linux-gnu
+ - os: ubuntu-22.04
+ packages:
+ processor: x86_64
+ system: Linux
+ target: x86_64-linux-gnu
+ name: llvm ${{ matrix.target }}
  outputs:
  cache-key: ${{ steps.cache-key.outputs.cache-key }}
  steps:
@@ -20,18 +54,18 @@ jobs:
  echo "sha=$value" >> "$GITHUB_OUTPUT"
 
  - id: cache-key
- run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
+ run: echo "cache-key-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
 
  - name: Cache
  id: cache-llvm
  uses: actions/cache@v4
  with:
  path: llvm-install
- key: ${{ steps.cache-key.outputs.cache-key }}
+ key: ${{ steps.cache-key.outputs[format('cache-key-{0}', matrix.target)] }}
  lookup-only: true
 
  - name: Install Tools
- if: steps.cache-llvm.outputs.cache-hit != 'true'
+ if: runner.os == 'Linux' && steps.cache-llvm.outputs.cache-hit != 'true'
  run: |
  set -euxo pipefail
  wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
@@ -41,7 +75,31 @@ jobs:
  sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
 
  sudo apt update
- sudo apt -y install cmake ninja-build clang lld
+ sudo apt -y install \
+ cmake \
+ ninja-build \
+ ${{ matrix.packages }}
+
+ - name: Install Tools
+ if: runner.os == 'macOS' && steps.cache-llvm.outputs.cache-hit != 'true'
+ run: |
+ set -euxo pipefail
+ brew update
+ brew install ninja
+
+ - name: Install LLVM
+ # We use the latest LLVM binaries to build our LLVM fork.
+ run: |
+ set -euxo pipefail
+ llvm_tarball_url=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+ https://api.github.com/repos/llvm/llvm-project/releases | jq -r \
+ 'first | .assets | map(select(.name |
+ endswith("${{ matrix.os-name }}-X64.tar.xz"))) | first |
+ .browser_download_url')
+ mkdir -p /tmp/llvm-upstream
+ wget -q -O - $llvm_tarball_url | tar -xJ --strip-components 1 \
+ -C /tmp/llvm-upstream
+ echo /tmp/llvm-upstream/bin >> $GITHUB_PATH
 
  - name: Checkout LLVM Source
  if: steps.cache-llvm.outputs.cache-hit != 'true'
@@ -60,42 +118,29 @@ jobs:
  -B llvm-build \
  -G Ninja \
  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+ -DCMAKE_ASM_COMPILER=clang \
+ -DCMAKE_ASM_COMPILER_TARGET="${{ matrix.target }}" \
  -DCMAKE_C_COMPILER=clang \
+ -DCMAKE_C_COMPILER_TARGET="${{ matrix.target }}" \
  -DCMAKE_CXX_COMPILER=clang++ \
+ -DCMAKE_CXX_COMPILER_TARGET="${{ matrix.target }}" \
  -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \
+ -DCMAKE_SYSTEM_NAME="${{ matrix.system }}" \
+ -DCMAKE_SYSTEM_PROCESSOR="${{ matrix.processor }}" \
  -DLLVM_BUILD_LLVM_DYLIB=ON \
  -DLLVM_ENABLE_ASSERTIONS=ON \
  -DLLVM_ENABLE_PROJECTS= \
  -DLLVM_ENABLE_RUNTIMES= \
+ -DLLVM_HOST_TRIPLE="${{ matrix.target }}" \
  -DLLVM_INSTALL_UTILS=ON \
  -DLLVM_LINK_LLVM_DYLIB=ON \
  -DLLVM_TARGETS_TO_BUILD=BPF \
  -DLLVM_USE_LINKER=lld
 
  - name: Install LLVM
  if: steps.cache-llvm.outputs.cache-hit != 'true'
- env:
- # Create symlinks rather than copies to conserve disk space. At the time of this writing,
- # GitHub-hosted runners have 14GB of SSD space
- # (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources).
- #
- # Since the LLVM build creates a bunch of symlinks (and this setting does not turn those
- # into symlinks-to-symlinks), use absolute symlinks so we can distinguish the two cases.
- CMAKE_INSTALL_MODE: ABS_SYMLINK
  run: cmake --build llvm-build --target install
 
- - name: Rewrite LLVM Symlinks
- if: steps.cache-llvm.outputs.cache-hit != 'true'
- # Move targets over the symlinks that point to them.
- #
- # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE.
- run: |
- set -euxo pipefail
- find llvm-install -type l -execdir sh -eux -c '
- for link in "$@"; do
- target=$(readlink "$link")
- case $target in
- /*) mv "$target" "$link" ;;
- esac
- done
- ' sh {} +
+ - name: Setup tmate session
+ if: ${{ failure() }}
+ uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,15 +9,63 @@ jobs:
  uses: ./.github/workflows/llvm.yml
 
  upload-bins:
- # TODO: Build for macos someday.
- runs-on: ubuntu-22.04
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ include:
+ - os: macos-13
+ target: aarch64-apple-darwin
+ target-llvm: aarch64-apple-darwin
+ - os: macos-13
+ target: x86_64-apple-darwin
+ target-llvm: x86_64-apple-darwin
+ - os: ubuntu-22.04
+ packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+ target: aarch64-unknown-linux-gnu
+ target-llvm: aarch64-linux-gnu
+ - os: ubuntu-22.04
+ packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+ target: aarch64-unknown-linux-musl
+ target-llvm: aarch64-linux-gnu
+ - os: ubuntu-22.04
+ packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross
+ target: riscv64-unknown-linux-gnu
+ target-llvm: riscv64-linux-gnu
+ - os: ubuntu-22.04
+ packages:
+ target: x86_64-unknown-linux-gnu
+ target-llvm: x86_64-linux-gnu
+ - os: ubuntu-22.04
+ packages:
+ target: x86_64-unknown-linux-musl
+ target-llvm: x86_64-linux-gnu
  needs: llvm
+
+ env:
+ CARGO_BUILD_TARGET: ${{ matrix.target }}
+ # We use the GNU sysroot as an LD path both for GNU and musl builds of
+ # bpf-linker - the user-space emulator and the path are used only for
+ # executing `llvm-config` and test binaries.
+ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+ CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+ CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+ LLVM_SYS_CROSS_LD_PREFIX: /usr/${{ matrix.target-llvm }}
+ RUST_BACKTRACE: full
+
  steps:
+ - name: Install Tools
+ if: runner.os == 'Linux'
+ run: |
+ set -euxo pipefail
+ sudo apt update
+ sudo apt -y install ${{ matrix.packages }}
+
  - name: Restore LLVM
  uses: actions/cache/restore@v4
  with:
  path: llvm-install
- key: ${{ needs.llvm.outputs.cache-key }}
+ key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target-llvm)] }}
  fail-on-cache-miss: true
 
  - name: Add LLVM to PATH

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -54,3 +54,4 @@ debug = true
 
 [patch.crates-io]
 compiletest_rs = { git = "https://github.com/Manishearth/compiletest-rs.git" }
+llvm-sys = { git = "https://gitlab.com/vadorovsky/llvm-sys.rs.git", branch = "cross-build" }