From 9081ec949870a943c2f14d40d2ecd8535aeab7ac Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Tue, 30 May 2023 18:40:40 +0100 Subject: [PATCH] chore(nix): Fix dependency caching (#1421) * chore: Move the git information to improve caching * change some rust code * cache changes * remove rust changes * Update .github/workflows/test.yml --- .github/workflows/test.yml | 2 +- flake.nix | 130 +++++++++++++++++++++++++------------ 2 files changed, 89 insertions(+), 43 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29f58ad519c..3854cfeaab3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,4 +58,4 @@ jobs: - name: Export cache from nix store if: steps.nix-store-cache.outputs.cache-hit != 'true' run: | - nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#cargo-artifacts + nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#native-cargo-artifacts diff --git a/flake.nix b/flake.nix index 696c38623ed..c02dbe14542 100644 --- a/flake.nix +++ b/flake.nix @@ -68,28 +68,41 @@ craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; - environment = { + sharedEnvironment = { + # We enable backtraces on any failure for help with debugging + RUST_BACKTRACE = "1"; + }; + + nativeEnvironment = sharedEnvironment // { # rust-bindgen needs to know the location of libclang LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + }; + wasmEnvironment = sharedEnvironment // { + # We set the environment variable because barretenberg must be compiled in a special way for wasm + BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin"; + }; + + testEnvironment = sharedEnvironment // { # Barretenberg fails if tests are run on multiple threads, so we set the test thread # count to 1 throughout the entire project # # Note: Setting this allows for consistent behavior across build and shells, but is mostly # hidden from the developer - i.e. when they see the command being run via `nix flake check` RUST_TEST_THREADS = "1"; - - # We enable backtraces on any failure for help with debugging - RUST_BACKTRACE = "1"; - - # We set the environment variable because barretenberg must be compiled in a special way for wasm - BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin"; }; # The `self.rev` property is only available when the working tree is not dirty GIT_COMMIT = if (self ? rev) then self.rev else "unknown"; GIT_DIRTY = if (self ? rev) then "false" else "true"; + # We use `.nr` and `.toml` files in tests so we need to create a special source + # filter to include those files in addition to usual rust/cargo source files + noirFilter = path: _type: builtins.match ".*nr$" path != null; + tomlFilter = path: _type: builtins.match ".*toml$" path != null; + sourceFilter = path: type: + (noirFilter path type) || (tomlFilter path type) || (craneLib.filterCargoSources path type); + # As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses # gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++ stdenv = @@ -98,21 +111,35 @@ else pkgs.llvmPackages.stdenv; - # Combine the environment and other configuration needed for crane to build our Rust packages - commonArgs = environment // { - pname = "noir"; + extraBuildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [ + # Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156 + pkgs.libiconv + pkgs.darwin.apple_sdk.frameworks.Security + ]; + + sharedArgs = { # x-release-please-start-version version = "0.6.0"; # x-release-please-end - # Use our custom stdenv to build and test our Rust project - inherit stdenv; + src = pkgs.lib.cleanSourceWith { + src = craneLib.path ./.; + filter = sourceFilter; + }; - src = ./.; + # TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell. + cargoClippyExtraArgs = "--all-targets -- -D warnings"; - # Running checks don't do much more than compiling itself and increase - # the build time by a lot, so we disable them throughout all our flakes - doCheck = false; + # TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell. + cargoTestExtraArgs = "--workspace"; + }; + + # Combine the environment and other configuration needed for crane to build our Rust packages + nativeArgs = nativeEnvironment // sharedArgs // { + pname = "noir-native"; + + # Use our custom stdenv to build and test our Rust project + inherit stdenv; nativeBuildInputs = [ # This provides the pkg-config tool to find barretenberg & other native libraries @@ -124,18 +151,21 @@ buildInputs = [ pkgs.llvmPackages.openmp pkgs.barretenberg - ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ - # Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156 - pkgs.libiconv - pkgs.darwin.apple_sdk.frameworks.Security - ]; + ] ++ extraBuildInputs; + }; + + # Combine the environment and other configuration needed for crane to build with the wasm feature + wasmArgs = wasmEnvironment // sharedArgs // { + pname = "noir-wasm"; - inherit GIT_COMMIT; - inherit GIT_DIRTY; + # We disable the default "plonk_bn254" feature and enable the "plonk_bn254_wasm" feature + cargoExtraArgs = "--no-default-features --features='plonk_bn254_wasm'"; + + buildInputs = [ ] ++ extraBuildInputs; }; # The `port` is parameterized to support parallel test runs without colliding static servers - testArgs = port: { + testArgs = port: testEnvironment // { # We provide `barretenberg-transcript00` from the overlay to the tests as a URL hosted via a static server # This is necessary because the Nix sandbox has no network access and downloading during tests would fail TRANSCRIPT_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf pkgs.barretenberg-transcript00}"; @@ -160,36 +190,52 @@ }; # Build *just* the cargo dependencies, so we can reuse all of that work between runs - cargoArtifacts = craneLib.buildDepsOnly commonArgs; + native-cargo-artifacts = craneLib.buildDepsOnly nativeArgs; + wasm-cargo-artifacts = craneLib.buildDepsOnly wasmArgs; + + noir-native = craneLib.buildPackage (nativeArgs // { + inherit GIT_COMMIT GIT_DIRTY; - noir = craneLib.buildPackage (commonArgs // { - inherit cargoArtifacts; + cargoArtifacts = native-cargo-artifacts; + + # We don't want to run checks or tests when just building the project + doCheck = false; + }); + + noir-wasm = craneLib.buildPackage (wasmArgs // { + inherit GIT_COMMIT GIT_DIRTY; + + cargoArtifacts = wasm-cargo-artifacts; + + # We don't want to run checks or tests when just building the project + doCheck = false; }); in rec { checks = { - cargo-clippy = craneLib.cargoClippy (commonArgs // { - inherit cargoArtifacts; + cargo-clippy = craneLib.cargoClippy (nativeArgs // { + inherit GIT_COMMIT GIT_DIRTY; - # TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell. - cargoClippyExtraArgs = "--all-targets -- -D warnings"; + cargoArtifacts = native-cargo-artifacts; }); - cargo-test = craneLib.cargoTest (commonArgs // (testArgs 8000) // { - inherit cargoArtifacts; - - # TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell. - cargoTestExtraArgs = "--workspace"; + cargo-test = craneLib.cargoTest (nativeArgs // (testArgs 8000) // { + inherit GIT_COMMIT GIT_DIRTY; - # It's unclear why doCheck needs to be enabled for tests to run but not clippy - doCheck = true; + cargoArtifacts = native-cargo-artifacts; }); }; - packages.default = noir; + packages = { + default = noir-native; - # We expose the `cargo-artifacts` derivation so we can cache our cargo dependencies in CI - packages.cargo-artifacts = cargoArtifacts; + inherit noir-native; + inherit noir-wasm; + + # We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI + inherit native-cargo-artifacts; + inherit wasm-cargo-artifacts; + }; # TODO(#1197): Look into installable apps with Nix flakes # apps.default = flake-utils.lib.mkApp { drv = nargo; }; @@ -197,7 +243,7 @@ # Setup the environment to match the stdenv from `nix build` & `nix flake check`, and # combine it with the environment settings, the inputs from our checks derivations, # and extra tooling via `nativeBuildInputs` - devShells.default = pkgs.mkShell.override { inherit stdenv; } (environment // { + devShells.default = pkgs.mkShell.override { inherit stdenv; } (nativeEnvironment // wasmEnvironment // testEnvironment // { inputsFrom = builtins.attrValues checks; nativeBuildInputs = with pkgs; [