Bootstrappable FCMP++

.github/workflows/build.yml

@@ -87,7 +87,7 @@ jobs:
       image: archlinux:latest
     steps:
       - name: install dependencies
-        run: pacman -Syyu --noconfirm base-devel git cmake boost openssl zeromq unbound libsodium readline expat gtest python3 doxygen graphviz hidapi libusb protobuf
+        run: pacman -Syyu --noconfirm base-devel git rust cmake boost openssl zeromq unbound libsodium readline expat gtest python3 doxygen graphviz hidapi libusb protobuf
       - name: configure git
         run: git config --global --add safe.directory '*'
       - uses: actions/checkout@v4

contrib/guix/guix-build

@@ -263,6 +263,29 @@ mkdir -p "$OUTDIR_BASE"
 LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}"
 mkdir -p "$LOGDIR_BASE"
 
+# Download and archive Rust dependencies.
+# Version and Hash need to be updated when:
+# - we bump the time-machine and Guix has a new version of Rust.
+# - Cargo.lock in src/fcmp_pp/fcmp_pp_rust is updated.
+RUST_DEPS_VERSION=0
+RUST_DEPS_HASH="0wi6a4drxg695b00k8kzxbz2vw6y9izp"
+RUST_DEPS_ARCHIVE="rust_deps-${RUST_DEPS_VERSION}.tar.gz"
+RUST_DEPS_STORE_ITEM="/gnu/store/${RUST_DEPS_HASH}-${RUST_DEPS_ARCHIVE}"
+if [ ! -f "${RUST_DEPS_STORE_ITEM}" ]; then
+    time-machine environment --manifest="${PWD}/contrib/guix/rust/cargo.scm" \
+                             --container \
+                             --pure \
+                             --network \
+                             --no-cwd \
+                             --user="user" \
+                             --share="$PWD"=/monero \
+                             -- env RUST_DEPS_ARCHIVE="$RUST_DEPS_ARCHIVE" \
+                                    bash /monero/contrib/guix/rust/cargo.sh
+
+    time-machine download ${RUST_DEPS_ARCHIVE}
+    rm ${RUST_DEPS_ARCHIVE}
+fi
+
 # Download the depends sources now as we won't have internet access in the build
 # container
 for host in $HOSTS; do
@@ -437,6 +460,7 @@ EOF
                                  --share="$DISTSRC_BASE"=/distsrc-base \
                                  --share="$OUTDIR_BASE"=/outdir-base \
                                  --share="$LOGDIR_BASE"=/logdir-base \
+                                 --share="$RUST_DEPS_STORE_ITEM"=/rust-deps \
                                  --expose="$(git rev-parse --git-common-dir)" \
                                  ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
                                  ${BASE_CACHE:+--share="$BASE_CACHE"} \

contrib/guix/libexec/build.sh

@@ -320,6 +320,32 @@ case "$HOST" in
     *mingw*)  HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;;
 esac
 
+case "$HOST" in
+    *darwin*)  ;;
+    *)  LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${NATIVE_GCC}/lib" ;;
+esac
+
+# error: "/gnu/store/<...>-rust-1.82.0/lib/rustlib/src/rust/library/Cargo.lock" does not exist,
+# unable to build with the standard library
+#
+# The standard library does not exist at the location Cargo expects.
+#
+# We can override the path to the Rust source by setting the __CARGO_TESTS_ONLY_SRC_ROOT environment variable.
+# See: https://github.com/rust-lang/cargo/blob/rust-1.82.0/src/cargo/core/compiler/standard_lib.rs#L183
+export __CARGO_TESTS_ONLY_SRC_ROOT=/rust/library
+
+# error: the `-Z` flag is only accepted on the nightly channel of Cargo, but this is the `stable` channel
+#
+# Since we don't have access to the nightly channel, we need to bypass the check with RUSTC_BOOTSTRAP.
+#
+# We could avoid using `-Z build-std` by cross-compiling the full standard library for each target. This approach
+# adds hours to our build time and greatly increases the amount of foreign source code that is compiled as part of
+# our build process.
+export RUSTC_BOOTSTRAP=1
+
+# See: https://rust-lang.github.io/rust-project-goals/2025h1/build-std.html
+CARGO_OPTIONS="-Zbuild-std=std,panic_abort;"
+
 export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
 # Force Trezor support for release binaries
 export USE_DEVICE_TREZOR_MANDATORY=1
@@ -347,16 +373,26 @@ mkdir -p "$DISTSRC"
     # checked out before starting a build.
     CMAKEFLAGS+=" -DMANUAL_SUBMODULES=1"
 
+    # Make sure cargo knows where to find the vendored sources.
+    mkdir -p /home/user/.cargo
+    cp contrib/guix/rust/config.toml /home/user/.cargo/
+    sed -i "s/TARGET/${HOST}/g" /home/user/.cargo/config.toml
+
+    # Unpack rust dependencies
+    mkdir -p /rust
+    tar xf /rust-deps -C /rust
+
     # Configure this DISTSRC for $HOST
     # shellcheck disable=SC2086
     env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
     cmake --toolchain "${BASEPREFIX}/${HOST}/share/toolchain.cmake" -S . -B build \
       -DCMAKE_INSTALL_PREFIX="${INSTALLPATH}" \
       -DCMAKE_EXE_LINKER_FLAGS="${HOST_LDFLAGS}" \
       -DCMAKE_SHARED_LINKER_FLAGS="${HOST_LDFLAGS}" \
+      -DCARGO_OPTIONS="${CARGO_OPTIONS}" \
       ${CMAKEFLAGS}
 
-    make -C build --jobs="$JOBS"
+    LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" make -C build --jobs="$JOBS"
 
     # Copy docs
     cp README.md LICENSE docs/ANONYMITY_NETWORKS.md "${INSTALLPATH}"

contrib/guix/manifest.scm

@@ -18,6 +18,7 @@
              (gnu packages perl)
              (gnu packages pkg-config)
              ((gnu packages python) #:select (python-minimal))
+             (gnu packages rust)
              ((gnu packages tls) #:select (openssl))
              ((gnu packages version-control) #:select (git-minimal))
              (guix build-system gnu)
@@ -275,6 +276,8 @@ chain for " target " development."))
         automake
         pkg-config
         cmake-minimal
+        rust
+        (list rust "cargo")
 
         ;; Scripting
         perl           ; required to build openssl in depends

contrib/guix/rust/cargo.scm

@@ -0,0 +1,72 @@
+(use-modules (gnu packages)
+             ((gnu packages bash) #:select (bash-minimal))
+             ((gnu packages certs) #:select (nss-certs))
+             (gnu packages compression)
+             (gnu packages curl)
+             (gnu packages moreutils)
+             (gnu packages rust)
+             ((gnu packages tls) #:select (openssl))
+             ((gnu packages web) #:select (jq))
+             (guix build-system trivial)
+             (guix download)
+             ((guix licenses) #:prefix license:)
+             (guix packages))
+
+;; We don't use (list rust "rust-src") for two reasons:
+;;
+;; - Hashes in Cargo.lock are replaced, resulting in:
+;;   error: checksum for `<package> <version>` changed between lock files
+;;
+;; - It drags in a bunch of unnecessary deps, including python.
+;;   See: guix graph --type=references rust | xdot -
+
+;; Instead, we create a new package with the unaltered rust source
+;; and vendor the standard library in cargo.sh
+
+(define-public rust-std
+  (package
+    (name "rust-std")
+    (version (package-version rust))
+    ;; You'd expect (source (package-source (rust)) to work here,
+    ;; but it refers to the source store item and NOT the .tar.gz archive
+    (source (origin
+              (method url-fetch)
+              (uri (origin-uri (package-source rust)))
+              (sha256
+                (content-hash-value (origin-hash (package-source rust))))))
+    (build-system trivial-build-system)
+    (native-inputs (list tar gzip))
+    (arguments
+      `(#:modules ((guix build utils))
+         #:builder
+         (begin
+           (use-modules (guix build utils))
+           (let ((out (assoc-ref %outputs "out"))
+                  (source (assoc-ref %build-inputs "source"))
+                  (tar (search-input-file %build-inputs "/bin/tar"))
+                  (gzip (search-input-file %build-inputs "/bin/gzip"))
+                  (gzip-path (string-append (assoc-ref %build-inputs "gzip") "/bin")))
+             (setenv "PATH" gzip-path)
+             (mkdir out)
+             (invoke tar "xvf" source "-C" out "--strip-components=1")))))
+    (synopsis (package-synopsis rust))
+    (description (package-description rust))
+    (home-page (package-home-page rust))
+    (license (package-license rust))))
+
+(packages->manifest
+  (append
+    (list
+      bash-minimal
+      coreutils-minimal
+      curl
+      findutils ;; find
+      grep
+      gzip
+      jq
+      nss-certs
+      openssl
+      sed
+      tar
+      (list rust "cargo")
+      rust-std)))

contrib/guix/rust/cargo.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+
+# Environment variables for determinism
+export LC_ALL=C
+export SOURCE_DATE_EPOCH=1397818193
+export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name"
+export TZ="UTC"
+
+# Given a package name and an output name, return the path of that output in our
+# current guix environment
+store_path() {
+    grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \
+        | head --lines=1 \
+        | sed --expression='s|\x29*$||' \
+              --expression='s|^[[:space:]]*"||' \
+              --expression='s|"[[:space:]]*$||'
+}
+
+echo "Fetching rust dependencies.."
+
+cd /monero/src/fcmp_pp/fcmp_pp_rust
+
+# error: the cargo feature `public-dependency` requires a nightly version of Cargo, but this is the `stable` channel
+#
+# https://doc.rust-lang.org/cargo/reference/unstable.html#public-dependency
+export RUSTC_BOOTSTRAP=1
+
+# Assert that `Cargo.lock` will remain unchanged
+CARGO_OPTIONS="--locked"
+
+# https://github.com/rust-lang/wg-cargo-std-aware/issues/23#issuecomment-1445119470
+# If we don't vendor std, we'll run into: 'error: no matching package named `compiler_builtins` found' during build.
+CARGO_OPTIONS+=" --sync $(store_path rust-std)/library/Cargo.toml"
+
+# Vendor fcmp_pp_rust + std library deps
+cargo vendor ${CARGO_OPTIONS} /rust/vendor
+
+cp -r "$(store_path rust-std)/library" /rust/library
+
+cd /rust/vendor
+
+# `cargo vendor` includes dozens of packages that aren't needed to build the standard library.
+# We can't simply remove these packages, because cargo expects them to be there.
+# Instead, we replace the packages with a stub, which is sufficient to pass cargo's checks.
+while IFS= read -r line; do
+    cd "$line"
+    find . -not -path "." -not -name "Cargo.toml" -not -name ".cargo-checksum.json" -delete
+    mkdir src
+    touch src/lib.rs
+
+    # Cargo.toml must remain unaltered.
+    # src/lib.rs must exist, but may be empty.
+    # 'e3b0...b855' is equivalent to `echo -n "" | sha256sum -`
+    # We can't set 'package' to the empty hash, as this might conflict with fcmp_pp_rust's Cargo.lock, resulting
+    # in the same error.
+    cat .cargo-checksum.json \
+      | jq '{files: {"Cargo.toml": .files["Cargo.toml"]}, package}' \
+      | jq '.files += {"src/lib.rs": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}' \
+      > .cargo-checksum.json.tmp
+    mv .cargo-checksum.json.tmp .cargo-checksum.json
+    cd ..
+done < "/monero/contrib/guix/rust/stubs"
+
+cd /rust
+
+# Create deterministic archive
+find . -print0 \
+  | sort --zero-terminated \
+  | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+  | gzip -9n > "/monero/$RUST_DEPS_ARCHIVE"

contrib/guix/rust/config.toml

@@ -0,0 +1,14 @@
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source."git+https://github.com/kayabaNerve/crypto-bigint?branch=c-repr"]
+git = "https://github.com/kayabaNerve/crypto-bigint"
+branch = "c-repr"
+replace-with = "vendored-sources"
+
+[source."git+https://github.com/kayabaNerve/fcmp-plus-plus"]
+git = "https://github.com/kayabaNerve/fcmp-plus-plus"
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "/rust/vendor"

contrib/guix/rust/stubs

@@ -0,0 +1,24 @@
+allocator-api2
+cc
+dlmalloc
+fortanix-sgx-abi
+getopts
+gimli-0.28.1
+hermit-abi
+r-efi
+r-efi-alloc
+rand
+rand_xorshift
+unicode-width
+unwinding
+wasi
+windows-sys
+windows-targets
+windows_aarch64_gnullvm
+windows_aarch64_msvc
+windows_i686_gnu
+windows_i686_gnullvm
+windows_i686_msvc
+windows_x86_64_gnu
+windows_x86_64_gnullvm
+windows_x86_64_msvc

src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt

@@ -86,10 +86,10 @@ endif()
 set(RUST_TARGET "${RUST_ARCH}-${RUST_PLATFORM}${RUST_TOOLCHAIN}")
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-  set(CARGO_CMD cargo build --target "${RUST_TARGET}")
+  set(CARGO_CMD cargo build --target "${RUST_TARGET}" ${CARGO_OPTIONS})
   set(TARGET_DIR "debug")
 else ()
-  set(CARGO_CMD cargo build --target "${RUST_TARGET}" --release)
+  set(CARGO_CMD cargo build --target "${RUST_TARGET}" --release ${CARGO_OPTIONS})
   set(TARGET_DIR "release")
 endif ()