From 176f1de6e635cedc3fc81b28f2883a1bc0354b94 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 22 Aug 2024 18:01:11 +0000 Subject: [PATCH 1/9] feat: Add reproducible build profile --- Cargo.toml | 7 +++++++ Makefile | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 9b525ff4e4aa..cb101fc3b8b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -266,6 +266,13 @@ inherits = "release" lto = "fat" codegen-units = 1 +[profile.reproducible] +inherits = "release" +debug = false +panic = "abort" +codegen-units = 1 +overflow-checks = true + [workspace.dependencies] # reth reth = { path = "bin/reth" } diff --git a/Makefile b/Makefile index 6b43c19e4f42..74c7198bc8ae 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,15 @@ install-op: ## Build and install the op-reth binary under `~/.cargo/bin`. build: ## Build the reth binary into `target` directory. cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)" +.PHONY: build-reproducible +build-reproducible: ## Build the reth binary into `target` directory with reproducible builds. + SOURCE_DATE_EPOCH=1724346102 \ + CARGO_INCREMENTAL=0 \ + LC_ALL=C \ + TZ=UTC \ + RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $$(pwd)=." \ + cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked + .PHONY: build-debug build-debug: ## Build the reth binary into `target/debug` directory. cargo build --bin reth --features "$(FEATURES)" From 954ab641c456354cbb1ff9fbff5e8159fd9dd6b3 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 23 Aug 2024 10:03:09 +0000 Subject: [PATCH 2/9] Add Dockerfile for reproducible builds --- Makefile | 2 +- reproducible.Dockerfile | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 reproducible.Dockerfile diff --git a/Makefile b/Makefile index 74c7198bc8ae..0756f8659073 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ build-reproducible: ## Build the reth binary into `target` directory with reprod CARGO_INCREMENTAL=0 \ LC_ALL=C \ TZ=UTC \ - RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $$(pwd)=." \ + RUSTFLAGS="-C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." \ cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked .PHONY: build-debug diff --git a/reproducible.Dockerfile b/reproducible.Dockerfile new file mode 100644 index 000000000000..bf89f2206a9b --- /dev/null +++ b/reproducible.Dockerfile @@ -0,0 +1,33 @@ +# Use the Rust 1.80 image based on Debian Bullseye +FROM rust:1.80-bullseye@sha256:c1490599f028ae06740706279a81c09687dde26c2d00f7160b85f63e9f6d8607 as builder + +# Install specific version of libclang-dev +RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 + +# Set environment variables for reproducibility +ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." +ENV SOURCE_DATE_EPOCH=1724346102 \ + CARGO_INCREMENTAL=0 \ + LC_ALL=C \ + TZ=UTC \ + RUSTFLAGS="$RUSTFLAGS" + +# Set the default features if not provided +ARG FEATURES="jemalloc asm-keccak" + +# Clone the repository at the specific branch +RUN git clone https://github.com/MoeMahhouk/reth /app +WORKDIR /app + +# Checkout the reproducible-build branch +RUN git checkout reproducible-build + +# Build the project with the reproducible settings +RUN cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked + +# Create a minimal final image with just the binary +FROM scratch as binaries + +# Copy the compiled binary from the builder stage +COPY --from=builder /app/target/reproducible/reth /reth + From 3a3432af7f3acd101ba39897d074a28a6d0b5fe8 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 11 Sep 2024 17:24:57 +0000 Subject: [PATCH 3/9] chore: refactor the timestamp and dockerfile name --- ...ible.Dockerfile => Dockerfile.reproducible | 26 +++++++++++-------- Makefile | 3 ++- 2 files changed, 17 insertions(+), 12 deletions(-) rename reproducible.Dockerfile => Dockerfile.reproducible (74%) diff --git a/reproducible.Dockerfile b/Dockerfile.reproducible similarity index 74% rename from reproducible.Dockerfile rename to Dockerfile.reproducible index bf89f2206a9b..92d854ed3bb3 100644 --- a/reproducible.Dockerfile +++ b/Dockerfile.reproducible @@ -4,30 +4,34 @@ FROM rust:1.80-bullseye@sha256:c1490599f028ae06740706279a81c09687dde26c2d00f7160 # Install specific version of libclang-dev RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 +# Clone the repository at the specific branch +RUN git clone https://github.com/MoeMahhouk/reth /app +WORKDIR /app + +# Checkout the reproducible-build branch +RUN git checkout reproducible-build + +# Get the latest commit timestamp and set SOURCE_DATE_EPOCH +RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment + # Set environment variables for reproducibility ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." -ENV SOURCE_DATE_EPOCH=1724346102 \ +ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ CARGO_INCREMENTAL=0 \ LC_ALL=C \ TZ=UTC \ - RUSTFLAGS="$RUSTFLAGS" + RUSTFLAGS="${RUSTFLAGS}" # Set the default features if not provided ARG FEATURES="jemalloc asm-keccak" -# Clone the repository at the specific branch -RUN git clone https://github.com/MoeMahhouk/reth /app -WORKDIR /app - -# Checkout the reproducible-build branch -RUN git checkout reproducible-build - # Build the project with the reproducible settings -RUN cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked +RUN . /etc/environment && \ + cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked # Create a minimal final image with just the binary FROM scratch as binaries # Copy the compiled binary from the builder stage COPY --from=builder /app/target/reproducible/reth /reth - diff --git a/Makefile b/Makefile index 0756f8659073..bdb9e8ba78df 100644 --- a/Makefile +++ b/Makefile @@ -61,9 +61,10 @@ install-op: ## Build and install the op-reth binary under `~/.cargo/bin`. build: ## Build the reth binary into `target` directory. cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)" +SOURCE_DATE_EPOCH := $(shell git log -1 --pretty=%ct) .PHONY: build-reproducible build-reproducible: ## Build the reth binary into `target` directory with reproducible builds. - SOURCE_DATE_EPOCH=1724346102 \ + SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ CARGO_INCREMENTAL=0 \ LC_ALL=C \ TZ=UTC \ From 48ffdf7beaa2415f23fe6b5ee024182cfb76a26f Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 22 Oct 2024 11:14:57 -0700 Subject: [PATCH 4/9] Update Dockerfile.reproducible --- Dockerfile.reproducible | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 92d854ed3bb3..ee43b046fc9d 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -5,7 +5,7 @@ FROM rust:1.80-bullseye@sha256:c1490599f028ae06740706279a81c09687dde26c2d00f7160 RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 # Clone the repository at the specific branch -RUN git clone https://github.com/MoeMahhouk/reth /app +RUN git clone https://github.com/paradigmxyz/reth /app WORKDIR /app # Checkout the reproducible-build branch From 3883d79cc4fa59af3989f21e267ca00d15fe8a56 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:08:07 -0400 Subject: [PATCH 5/9] fix: statically link binary --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0a13210fc351..5ad7abac6755 100644 --- a/Makefile +++ b/Makefile @@ -63,14 +63,14 @@ build: ## Build the reth binary into `target` directory. cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)" SOURCE_DATE_EPOCH := $(shell git log -1 --pretty=%ct) -.PHONY: build-reproducible -build-reproducible: ## Build the reth binary into `target` directory with reproducible builds. +.PHONY: reproducible +reproducible: ## Build the reth binary into `target` directory with reproducible builds. Only works for x86_64-unknown-linux-gnu currently SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ CARGO_INCREMENTAL=0 \ LC_ALL=C \ TZ=UTC \ - RUSTFLAGS="-C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." \ - cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked + RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." \ + cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu .PHONY: build-debug build-debug: ## Build the reth binary into `target/debug` directory. From f4a85dc9f9214f4a1ba15bf5e5d4a5108992071d Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:23:12 -0400 Subject: [PATCH 6/9] fix: use rust 1.82, add static args to dockerfile --- Dockerfile.reproducible | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index ee43b046fc9d..449d72587a7e 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -1,5 +1,5 @@ -# Use the Rust 1.80 image based on Debian Bullseye -FROM rust:1.80-bullseye@sha256:c1490599f028ae06740706279a81c09687dde26c2d00f7160b85f63e9f6d8607 as builder +# Use the Rust 1.82 image based on Debian Bullseye +FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af2672370559d7e854653c66d as builder # Install specific version of libclang-dev RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 @@ -16,7 +16,7 @@ RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment # Set environment variables for reproducibility -ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." +ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ CARGO_INCREMENTAL=0 \ LC_ALL=C \ @@ -27,8 +27,8 @@ ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ ARG FEATURES="jemalloc asm-keccak" # Build the project with the reproducible settings -RUN . /etc/environment && \ - cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked +RUN . /etc/environment && \ + cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu # Create a minimal final image with just the binary FROM scratch as binaries From 27d21c87838b70aaeda8a3876a72e1f4a2ef2a25 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:48:54 -0400 Subject: [PATCH 7/9] fix: use proper target dir for Dockerfile.reproducible --- Dockerfile.reproducible | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 449d72587a7e..bf4bfe1f73f4 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -34,4 +34,4 @@ RUN . /etc/environment && \ FROM scratch as binaries # Copy the compiled binary from the builder stage -COPY --from=builder /app/target/reproducible/reth /reth +COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/reth /reth From 9193098d636d951afa7606bcc5932e20a93f6116 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:00:01 -0400 Subject: [PATCH 8/9] fix: use same casing for FROM and AS --- Dockerfile.reproducible | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index bf4bfe1f73f4..ec11bc3ec3d6 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -1,5 +1,5 @@ # Use the Rust 1.82 image based on Debian Bullseye -FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af2672370559d7e854653c66d as builder +FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af2672370559d7e854653c66d AS builder # Install specific version of libclang-dev RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 From 0ff16184f91e4b41a02d1974e427d18850a3cfa8 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:13:38 -0400 Subject: [PATCH 9/9] fix: use single dollar sign for pwd --- Dockerfile.reproducible | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index ec11bc3ec3d6..12c12dd7c7d4 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -16,7 +16,7 @@ RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment # Set environment variables for reproducibility -ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." +ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $(pwd)=." ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ CARGO_INCREMENTAL=0 \ LC_ALL=C \ @@ -31,7 +31,7 @@ RUN . /etc/environment && \ cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu # Create a minimal final image with just the binary -FROM scratch as binaries +FROM scratch AS binaries # Copy the compiled binary from the builder stage COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/reth /reth