Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use portable builds by default #5489

Closed
wants to merge 14 commits into from
Closed

Use portable builds by default #5489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
28d0187
build 2 instead of 8 images
antondlr Mar 26, 2024
ac94dcd
syntax fix for multiple image tags
antondlr Mar 26, 2024
0ec722e
portable and force-adx are mutually exclusive
antondlr Mar 26, 2024
36d5c79
use `lld` linker for cross builds
antondlr Mar 26, 2024
5a6cd9c
pin lld major version
antondlr Mar 26, 2024
56c9935
cleanup
antondlr Mar 26, 2024
1b1ece8
cross-build `lcli`, multiarch
antondlr Mar 26, 2024
938ea40
syntax errors
antondlr Mar 26, 2024
6def1aa
dynamically set repo owner
antondlr Mar 26, 2024
e5c8fce
cleanup
antondlr Mar 26, 2024
8264d30
try with `format`
antondlr Mar 26, 2024
d1ea982
only build docker `portable`
antondlr Mar 26, 2024
b2abc50
scrap non-portable builds
antondlr Mar 26, 2024
e6a9753
rework-nondocker-builds
antondlr Mar 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 60 additions & 60 deletions .github/workflows/docker.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@ concurrency:
env:
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
IMAGE_NAME: ${{ github.repository_owner}}/lighthouse
LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli
# Enable self-hosted runners for the sigp repo only.
SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
# Enable self-hosted runners for your own repo only.
SELF_HOSTED_RUNNERS: ${{ github.repository == format('{0}/{1}',github.repository_owner ,'lighthouse') }}

jobs:
# Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX
Expand Down Expand Up @@ -49,27 +47,22 @@ jobs:
VERSION: ${{ env.VERSION }}
VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
build-docker-single-arch:
name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }}
# Use self-hosted runners only on the sigp repo.
runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }}
name: build-docker-${{ matrix.binary }}-${{ matrix.cpu_arch }}${{ matrix.features.version_suffix }}
# Use self-hosted runners only on your own repo.
runs-on: ${{ github.repository == format('{0}/{1}',github.repository_owner ,'lighthouse') && fromJson('["self-hosted", "linux", "X64", "large"]') || 'ubuntu-22.04' }}
strategy:
matrix:
binary: [aarch64,
aarch64-portable,
x86_64,
x86_64-portable]
features: [
{version_suffix: "", env: "gnosis,slasher-lmdb,slasher-mdbx,jemalloc"},
{version_suffix: "-dev", env: "jemalloc,spec-minimal"}
]
binary: [lighthouse,
lcli]
cpu_arch: [aarch64,
x86_64]
include:
- profile: maxperf

needs: [extract-version]
env:
VERSION: ${{ needs.extract-version.outputs.VERSION }}
VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
FEATURE_SUFFIX: ${{ matrix.features.version_suffix }}
steps:
- uses: actions/checkout@v4
- name: Update Rust
Expand All @@ -78,52 +71,80 @@ jobs:
- name: Dockerhub login
run: |
echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
- name: Cross build Lighthouse binary

- name: Sets env vars for Lighthouse
if: startsWith(matrix.binary, 'lighthouse')
run: |
echo "CROSS_FEATURES=gnosis,spec-minimal,slasher-lmdb,jemalloc" >> $GITHUB_ENV

- name: Set `make` command for lighthouse
if: startsWith(matrix.binary, 'lighthouse')
run: |
echo "MAKE_CMD=build-${{ matrix.cpu_arch }}-portable" >> $GITHUB_ENV
- name: Set `make` command for lcli
if: startsWith(matrix.binary, 'lcli')
run: |
echo "MAKE_CMD=build-lcli-${{ matrix.cpu_arch }}" >> $GITHUB_ENV

- name: Cross build binaries
run: |
cargo install cross
env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }}
env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ env.CROSS_FEATURES }} make ${{ env.MAKE_CMD }}

- name: Make bin dir
run: mkdir ./bin
- name: Move cross-built binary into Docker scope (if ARM)
if: startsWith(matrix.binary, 'aarch64')
run: mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
- name: Move cross-built binary into Docker scope (if x86_64)
if: startsWith(matrix.binary, 'x86_64')
run: mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
- name: Move cross-built binary into Docker scope
run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/${{ matrix.binary }} ./bin

- name: Map aarch64 to arm64 short arch
if: startsWith(matrix.binary, 'aarch64')
if: startsWith(matrix.cpu_arch, 'aarch64')
run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
- name: Map x86_64 to amd64 short arch
if: startsWith(matrix.binary, 'x86_64')
if: startsWith(matrix.cpu_arch, 'x86_64')
run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV;
- name: Set modernity suffix
if: endsWith(matrix.binary, '-portable') != true
run: echo "MODERNITY_SUFFIX=-modern" >> $GITHUB_ENV;

- name: Install QEMU
if: env.SELF_HOSTED_RUNNERS == 'false'
run: sudo apt-get update && sudo apt-get install -y qemu-user-static

- name: Set up Docker Buildx
if: env.SELF_HOSTED_RUNNERS == 'false'
uses: docker/setup-buildx-action@v3

- name: Build and push
- name: Build and push (Lighthouse)
if: startsWith(matrix.binary, 'lighthouse')
uses: docker/build-push-action@v5
with:
file: ./Dockerfile.cross
context: .
platforms: linux/${{ env.SHORT_ARCH }}
push: true
tags: ${{ env.IMAGE_NAME }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}${{ env.MODERNITY_SUFFIX }}${{ env.FEATURE_SUFFIX }}
tags: |
${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}
${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}-dev
${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}-modern
${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}-modern-dev

- name: Build and push (lcli)
if: startsWith(matrix.binary, 'lcli')
uses: docker/build-push-action@v5
with:
file: ./lcli/Dockerfile.cross
context: .
platforms: linux/${{ env.SHORT_ARCH }}
push: true

tags: |
${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}


build-docker-multiarch:
name: build-docker-multiarch${{ matrix.modernity }}
name: build-docker-${{ matrix.binary }}-multiarch
runs-on: ubuntu-22.04
needs: [build-docker-single-arch, extract-version]
strategy:
matrix:
modernity: ["", "-modern"]
binary: [lighthouse,
lcli]
needs: [build-docker-single-arch, extract-version]
env:
VERSION: ${{ needs.extract-version.outputs.VERSION }}
VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
Expand All @@ -135,29 +156,8 @@ jobs:
run: |
echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin

- name: Create and push multiarch manifest
run: |
docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};

build-docker-lcli:
runs-on: ubuntu-22.04
needs: [extract-version]
env:
VERSION: ${{ needs.extract-version.outputs.VERSION }}
VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
steps:
- uses: actions/checkout@v4
- name: Dockerhub login
- name: Create and push multiarch manifests
run: |
echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
- name: Build lcli and push
uses: docker/build-push-action@v5
with:
build-args: |
FEATURES=portable
context: .
push: true
file: ./lcli/Dockerfile
tags: ${{ env.LCLI_IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }}
docker buildx imagetools create -t ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}${VERSION_SUFFIX} \
${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-arm64${VERSION_SUFFIX} \
${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-amd64${VERSION_SUFFIX};
88 changes: 36 additions & 52 deletions .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ env:
REPO_NAME: ${{ github.repository_owner }}/lighthouse
IMAGE_NAME: ${{ github.repository_owner }}/lighthouse
# Enable self-hosted runners for the sigp repo only.
SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
SELF_HOSTED_RUNNERS: ${{ github.repository == format('{0}/{1}',github.repository_owner ,'lighthouse') }}

jobs:
extract-version:
Expand All @@ -31,37 +31,21 @@ jobs:
strategy:
matrix:
arch: [aarch64-unknown-linux-gnu,
aarch64-unknown-linux-gnu-portable,
x86_64-unknown-linux-gnu,
x86_64-unknown-linux-gnu-portable,
x86_64-apple-darwin,
x86_64-apple-darwin-portable,
x86_64-windows,
x86_64-windows-portable]
x86_64-windows]
include:
- arch: aarch64-unknown-linux-gnu
runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }}
profile: maxperf
- arch: aarch64-unknown-linux-gnu-portable
runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }}
runner: ${{ github.repository == format('{0}/{1}',github.repository_owner ,'lighthouse') && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }}
profile: maxperf
- arch: x86_64-unknown-linux-gnu
runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }}
profile: maxperf
- arch: x86_64-unknown-linux-gnu-portable
runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }}
runner: ${{ github.repository == format('{0}/{1}',github.repository_owner ,'lighthouse') && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }}
profile: maxperf
- arch: x86_64-apple-darwin
runner: macos-latest
profile: maxperf
- arch: x86_64-apple-darwin-portable
runner: macos-latest
profile: maxperf
- arch: x86_64-windows
runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }}
profile: maxperf
- arch: x86_64-windows-portable
runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }}
runner: ${{ github.repository == format('{0}/{1}',github.repository_owner ,'lighthouse') && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }}
profile: maxperf

runs-on: ${{ matrix.runner }}
Expand Down Expand Up @@ -90,53 +74,29 @@ jobs:
# Builds
# ==============================

- name: Build Lighthouse for aarch64-unknown-linux-gnu-portable
if: matrix.arch == 'aarch64-unknown-linux-gnu-portable'
run: |
cargo install cross
env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64-portable

- name: Build Lighthouse for aarch64-unknown-linux-gnu
if: matrix.arch == 'aarch64-unknown-linux-gnu'
run: |
cargo install cross
env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64

- name: Build Lighthouse for x86_64-unknown-linux-gnu-portable
if: matrix.arch == 'x86_64-unknown-linux-gnu-portable'
run: |
cargo install cross
env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64-portable
env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64-portable

- name: Build Lighthouse for x86_64-unknown-linux-gnu
if: matrix.arch == 'x86_64-unknown-linux-gnu'
run: |
cargo install cross
env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64

- name: Move cross-compiled binary
if: startsWith(matrix.arch, 'aarch64')
run: mv target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse
env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64-portable

- name: Move cross-compiled binary
if: startsWith(matrix.arch, 'x86_64-unknown-linux-gnu')
run: mv target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse

- name: Build Lighthouse for x86_64-apple-darwin portable
if: matrix.arch == 'x86_64-apple-darwin-portable'
run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}
if: contains(matrix.arch, 'unknown-linux-gnu')
run: mv target/${{ matrix.arch }}/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse

- name: Build Lighthouse for x86_64-apple-darwin modern
- name: Build Lighthouse for x86_64-apple-darwin
if: matrix.arch == 'x86_64-apple-darwin'
run: cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }}

- name: Build Lighthouse for Windows portable
if: matrix.arch == 'x86_64-windows-portable'
run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}

- name: Build Lighthouse for Windows modern
- name: Build Lighthouse for Windows
if: matrix.arch == 'x86_64-windows'
run: cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }}
run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}

- name: Configure GPG and create artifacts
if: startsWith(matrix.arch, 'x86_64-windows') != true
Expand All @@ -151,7 +111,14 @@ jobs:
cd artifacts
tar -czf lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz lighthouse
echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
for ext in "tar.gz" "tar.gz.asc";\
do for f in *.$ext;\
do cp $f "../${f%.$ext}-portable.$ext";\
done;\
done

mv *tar.gz* ..


- name: Configure GPG and create artifacts Windows
if: startsWith(matrix.arch, 'x86_64-windows')
Expand All @@ -166,6 +133,7 @@ jobs:
tar -czf lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz lighthouse.exe
gpg --passphrase "$env:GPG_PASSPHRASE" --batch --pinentry-mode loopback -ab lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
move *tar.gz* ..
# todo copy to `-portable`

# =======================================================================
# Upload artifacts
Expand All @@ -179,13 +147,29 @@ jobs:
path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
compression-level: 0

- name: Upload artifact (copy)
if: startsWith(matrix.arch, 'x86_64-windows') != true
uses: actions/upload-artifact@v4
with:
name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}-portable.tar.gz
path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}-portable.tar.gz
compression-level: 0

- name: Upload signature
uses: actions/upload-artifact@v4
with:
name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
compression-level: 0

- name: Upload signature (copy)
if: startsWith(matrix.arch, 'x86_64-windows') != true
uses: actions/upload-artifact@v4
with:
name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}-portable.tar.gz.asc
path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}-portable.tar.gz.asc
compression-level: 0

draft-release:
name: Draft Release
needs: [build, extract-version]
Expand Down
5 changes: 5 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ build-aarch64:
build-aarch64-portable:
cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked

build-lcli-x86_64:
cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked
build-lcli-aarch64:
cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked

# Create a `.tar.gz` containing a binary for a specific target.
define tarball_release_binary
cp $(1)/lighthouse $(BIN_DIR)/lighthouse
Expand Down
6 changes: 6 additions & 0 deletions lcli/Dockerfile.cross
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# This image is meant to enable cross-architecture builds.
# It assumes the lcli binary has already been
# compiled for `$TARGETPLATFORM` and moved to `./bin`.
FROM --platform=$TARGETPLATFORM ubuntu:22.04
RUN apt update && apt -y upgrade && apt clean && rm -rf /var/lib/apt/lists/*
COPY ./bin/lcli /usr/local/bin/lcli
Loading