ci: Build statically linked bpf-linker in CI

Build LLVM and bpf-linker with statically linked libLLVM for multiple architectures. Include the resulting build artifacts on the release pages, so they can be installed with `cargo binstall`
aya-rs · Sep 21, 2024 · 74d6ba5 · 74d6ba5
1 parent 07365b6
commit 74d6ba5
Show file tree

Hide file tree

Showing 5 changed files with 183 additions and 71 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -48,21 +48,58 @@ jobs:
         run: cargo fmt --all -- --check
 
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.target.os }}
     strategy:
       fail-fast: false
       matrix:
         rust:
           - stable
           - beta
           - nightly
-        llvm:
-          - 19
-          - source
+        target:
+          - os: macos-13
+            target: aarch64-apple-darwin
+            target-llvm: aarch64-apple-darwin
+          - os: macos-13
+            target: x86_64-apple-darwin
+            target-llvm: x86_64-apple-darwin
+          # We use the GNU builds of LLVM both for GNU and musl builds of
+          # bpf-linker - it doesn't affect the type of libc being picked by
+          # Cargo and musl Rust toolchains come with their own copies of
+          # libc.a.
+          - os: ubuntu-22.04
+            packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+            target: aarch64-unknown-linux-gnu
+            target-llvm: aarch64-linux-gnu
+          - os: ubuntu-22.04
+            packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+            target: aarch64-unknown-linux-musl
+            target-llvm: aarch64-linux-gnu
+          - os: ubuntu-22.04
+            packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross
+            target: riscv64gc-unknown-linux-gnu
+            target-llvm: riscv64-linux-gnu
+          - os: ubuntu-22.04
+            packages:
+            target: x86_64-unknown-linux-gnu
+            target-llvm: x86_64-linux-gnu
+          - os: ubuntu-22.04
+            packages:
+            target: x86_64-unknown-linux-musl
+            target-llvm: x86_64-linux-gnu
     name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }}
     needs: llvm
 
     env:
+      CARGO_BUILD_TARGET: ${{ matrix.target.target }}
+      # We use the GNU sysroot as an LD path both for GNU and musl builds of
+      # bpf-linker - the user-space emulator and the path are used only for
+      # executing `llvm-config` and test binaries.
+      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+      CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/riscv64-linux-gnu
+      CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/riscv64-linux-gnu
+      LLVM_SYS_CROSS_LD_PREFIX: /usr/${{ matrix.target.target-llvm }}
       RUST_BACKTRACE: full
 
     steps:
@@ -80,8 +117,7 @@ jobs:
         with:
           toolchain: ${{ matrix.rust }}
           components: rust-src
-          # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15.
-          targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl
+          targets: ${{ matrix.target.target }}
 
       - uses: Swatinem/rust-cache@v2
 
@@ -92,7 +128,7 @@ jobs:
         run: cargo build
 
       - name: Install dependencies
-        if: matrix.rust == 'nightly'
+        if: runner.os == 'Linux'
         # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
         # enum values which was added in clang 15[1].
         #
@@ -108,42 +144,19 @@ jobs:
           wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
           echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
           sudo apt update
-          sudo apt -y install clang gcc-multilib
-
-      - name: Install LLVM
-        if: matrix.llvm != 'source'
-        run: |
-          set -euxo pipefail
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list
-
-          sudo apt update
-          # TODO(vadorovsky): Remove the requirement of libpolly.
-          #
-          # Packages from apt.llvm.org are being built all at once, with one
-          # cmake build with superset of options, then different binaries and
-          # libraries are being included in different packages.
-          #
-          # That results in `llvm-config --libname --link-static` mentioning
-          # libpolly, even if it's not installed. The output of that command is
-          # being used in build.rs of llvm-sys, so building llvm-sys on such
-          # system is complaining about lack of libpolly.
-          #
-          # Hopefully that nightmare goes away once we switch to binstalls and
-          # ditch the system LLVM option.
-          sudo apt -y install llvm-${{ matrix.llvm }}-dev libpolly-${{ matrix.llvm }}-dev
-          echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH
+          sudo apt -y install \
+            clang \
+            gcc-multilib \
+            ${{ matrix.target.packages }}
 
       - name: Restore LLVM
-        if: matrix.llvm == 'source'
         uses: actions/cache/restore@v4
         with:
           path: llvm-install
-          key: ${{ needs.llvm.outputs.cache-key }}
+          key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target.target-llvm)] }}
           fail-on-cache-miss: true
 
       - name: Add LLVM to PATH && LD_LIBRARY_PATH
-        if: matrix.llvm == 'source'
         run: |
           set -euxo pipefail
           echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH

diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml
@@ -3,13 +3,52 @@ name: LLVM
 on:
   workflow_call:
     outputs:
-      cache-key:
-        value: ${{ jobs.llvm.outputs.cache-key }}
+      cache-key-aarch64-apple-darwin:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-apple-darwin }}
+      cache-key-x86_64-apple-darwin:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-apple-darwin }}
+      cache-key-aarch64-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-linux-gnu }}
+      cache-key-riscv64-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-riscv64-linux-gnu }}
+      cache-key-x86_64-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-linux-gnu }}
 
 jobs:
   llvm:
-    runs-on: ubuntu-22.04
-    name: llvm
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-13
+            os-name: macOS
+            processor: arm64
+            system: Darwin
+            target: aarch64-apple-darwin
+          - os: macos-13
+            os-name: macOS
+            processor: x86_64
+            system: Darwin
+            target: x86_64-apple-darwin
+          - os: ubuntu-22.04
+            os-name: Linux
+            packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+            processor: aarch64
+            system: Linux
+            target: aarch64-linux-gnu
+          - os: ubuntu-22.04
+            os-name: Linux
+            packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross
+            processor: riscv64
+            system: Linux
+            target: riscv64-linux-gnu
+          - os: ubuntu-22.04
+            os-name: Linux
+            packages:
+            processor: x86_64
+            system: Linux
+            target: x86_64-linux-gnu
+    name: llvm ${{ matrix.target }}
     outputs:
       cache-key: ${{ steps.cache-key.outputs.cache-key }}
     steps:
@@ -20,18 +59,18 @@ jobs:
           echo "sha=$value" >> "$GITHUB_OUTPUT"
 
       - id: cache-key
-        run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
+        run: echo "cache-key-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
 
       - name: Cache
         id: cache-llvm
         uses: actions/cache@v4
         with:
           path: llvm-install
-          key: ${{ steps.cache-key.outputs.cache-key }}
+          key: ${{ steps.cache-key.outputs[format('cache-key-{0}', matrix.target)] }}
           lookup-only: true
 
       - name: Install Tools
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+        if: runner.os == 'Linux' && steps.cache-llvm.outputs.cache-hit != 'true'
         run: |
           set -euxo pipefail
           wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
@@ -41,7 +80,31 @@ jobs:
             sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
 
           sudo apt update
-          sudo apt -y install cmake ninja-build clang lld
+          sudo apt -y install \
+            cmake \
+            ninja-build \
+            ${{ matrix.packages }}
+
+      - name: Install Tools
+        if: runner.os == 'macOS' && steps.cache-llvm.outputs.cache-hit != 'true'
+        run: |
+          set -euxo pipefail
+          brew update
+          brew install ninja
+
+      - name: Install LLVM
+        # We use the latest LLVM binaries to build our LLVM fork.
+        run: |
+          set -euxo pipefail
+          llvm_tarball_url=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            https://api.github.com/repos/llvm/llvm-project/releases | jq -r \
+            'first | .assets | map(select(.name |
+             endswith("${{ matrix.os-name }}-X64.tar.xz"))) | first |
+             .browser_download_url')
+          mkdir -p /tmp/llvm-upstream
+          wget -q -O - $llvm_tarball_url | tar -xJ --strip-components 1 \
+            -C /tmp/llvm-upstream
+          echo /tmp/llvm-upstream/bin >> $GITHUB_PATH
 
       - name: Checkout LLVM Source
         if: steps.cache-llvm.outputs.cache-hit != 'true'
@@ -60,42 +123,29 @@ jobs:
             -B llvm-build \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DCMAKE_ASM_COMPILER=clang \
+            -DCMAKE_ASM_COMPILER_TARGET="${{ matrix.target }}" \
             -DCMAKE_C_COMPILER=clang \
+            -DCMAKE_C_COMPILER_TARGET="${{ matrix.target }}" \
             -DCMAKE_CXX_COMPILER=clang++ \
+            -DCMAKE_CXX_COMPILER_TARGET="${{ matrix.target }}" \
             -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \
+            -DCMAKE_SYSTEM_NAME="${{ matrix.system }}" \
+            -DCMAKE_SYSTEM_PROCESSOR="${{ matrix.processor }}" \
             -DLLVM_BUILD_LLVM_DYLIB=ON \
             -DLLVM_ENABLE_ASSERTIONS=ON \
             -DLLVM_ENABLE_PROJECTS= \
             -DLLVM_ENABLE_RUNTIMES= \
+            -DLLVM_HOST_TRIPLE="${{ matrix.target }}" \
             -DLLVM_INSTALL_UTILS=ON \
             -DLLVM_LINK_LLVM_DYLIB=ON \
             -DLLVM_TARGETS_TO_BUILD=BPF \
             -DLLVM_USE_LINKER=lld
 
       - name: Install LLVM
         if: steps.cache-llvm.outputs.cache-hit != 'true'
-        env:
-          # Create symlinks rather than copies to conserve disk space. At the time of this writing,
-          # GitHub-hosted runners have 14GB of SSD space
-          # (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources).
-          #
-          # Since the LLVM build creates a bunch of symlinks (and this setting does not turn those
-          # into symlinks-to-symlinks), use absolute symlinks so we can distinguish the two cases.
-          CMAKE_INSTALL_MODE: ABS_SYMLINK
         run: cmake --build llvm-build --target install
 
-      - name: Rewrite LLVM Symlinks
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
-        # Move targets over the symlinks that point to them.
-        #
-        # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE.
-        run: |
-          set -euxo pipefail
-          find llvm-install -type l -execdir sh -eux -c '
-            for link in "$@"; do
-              target=$(readlink "$link")
-              case $target in
-                /*) mv "$target" "$link" ;;
-              esac
-            done
-          ' sh {} +
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,15 +9,63 @@ jobs:
     uses: ./.github/workflows/llvm.yml
 
   upload-bins:
-    # TODO: Build for macos someday.
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-13
+            target: aarch64-apple-darwin
+            target-llvm: aarch64-apple-darwin
+          - os: macos-13
+            target: x86_64-apple-darwin
+            target-llvm: x86_64-apple-darwin
+          - os: ubuntu-22.04
+            packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+            target: aarch64-unknown-linux-gnu
+            target-llvm: aarch64-linux-gnu
+          - os: ubuntu-22.04
+            packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
+            target: aarch64-unknown-linux-musl
+            target-llvm: aarch64-linux-gnu
+          - os: ubuntu-22.04
+            packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross
+            target: riscv64-unknown-linux-gnu
+            target-llvm: riscv64-linux-gnu
+          - os: ubuntu-22.04
+            packages:
+            target: x86_64-unknown-linux-gnu
+            target-llvm: x86_64-linux-gnu
+          - os: ubuntu-22.04
+            packages:
+            target: x86_64-unknown-linux-musl
+            target-llvm: x86_64-linux-gnu
     needs: llvm
+
+    env:
+      CARGO_BUILD_TARGET: ${{ matrix.target }}
+      # We use the GNU sysroot as an LD path both for GNU and musl builds of
+      # bpf-linker - the user-space emulator and the path are used only for
+      # executing `llvm-config` and test binaries.
+      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+      CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+      CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu
+      LLVM_SYS_CROSS_LD_PREFIX: /usr/${{ matrix.target-llvm }}
+      RUST_BACKTRACE: full
+
     steps:
+      - name: Install Tools
+        if: runner.os == 'Linux'
+        run: |
+          set -euxo pipefail
+          sudo apt update
+          sudo apt -y install ${{ matrix.packages }}
+
       - name: Restore LLVM
         uses: actions/cache/restore@v4
         with:
           path: llvm-install
-          key: ${{ needs.llvm.outputs.cache-key }}
+          key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target-llvm)] }}
           fail-on-cache-miss: true
 
       - name: Add LLVM to PATH

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -54,3 +54,4 @@ debug = true
 
 [patch.crates-io]
 compiletest_rs = { git = "https://github.com/Manishearth/compiletest-rs.git" }
+llvm-sys = { git = "https://gitlab.com/vadorovsky/llvm-sys.rs.git", branch = "cross-build" }