From 2a8a40cbb8bed6d2352e23b8e8fd975831cd5105 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Thu, 1 Jun 2023 19:57:28 +0100 Subject: [PATCH] Revert "chore(nix): Fix dependency caching (#1421)" This reverts commit 9081ec949870a943c2f14d40d2ecd8535aeab7ac. --- .github/workflows/test.yml | 2 +- flake.nix | 130 ++++++++++++------------------------- 2 files changed, 43 insertions(+), 89 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3854cfeaab..29f58ad519 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,4 +58,4 @@ jobs: - name: Export cache from nix store if: steps.nix-store-cache.outputs.cache-hit != 'true' run: | - nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#cargo-artifacts diff --git a/flake.nix b/flake.nix index c02dbe1454..696c38623e 100644 --- a/flake.nix +++ b/flake.nix @@ -68,41 +68,28 @@ craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; - sharedEnvironment = { - # We enable backtraces on any failure for help with debugging - RUST_BACKTRACE = "1"; - }; - - nativeEnvironment = sharedEnvironment // { + environment = { # rust-bindgen needs to know the location of libclang LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; - }; - wasmEnvironment = sharedEnvironment // { - # We set the environment variable because barretenberg must be compiled in a special way for wasm - BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin"; - }; - - testEnvironment = sharedEnvironment // { # Barretenberg fails if tests are run on multiple threads, so we set the test thread # count to 1 throughout the entire project # # Note: Setting this allows for consistent behavior across build and shells, but is mostly # hidden from the developer - i.e. when they see the command being run via `nix flake check` RUST_TEST_THREADS = "1"; + + # We enable backtraces on any failure for help with debugging + RUST_BACKTRACE = "1"; + + # We set the environment variable because barretenberg must be compiled in a special way for wasm + BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin"; }; # The `self.rev` property is only available when the working tree is not dirty GIT_COMMIT = if (self ? rev) then self.rev else "unknown"; GIT_DIRTY = if (self ? rev) then "false" else "true"; - # We use `.nr` and `.toml` files in tests so we need to create a special source - # filter to include those files in addition to usual rust/cargo source files - noirFilter = path: _type: builtins.match ".*nr$" path != null; - tomlFilter = path: _type: builtins.match ".*toml$" path != null; - sourceFilter = path: type: - (noirFilter path type) || (tomlFilter path type) || (craneLib.filterCargoSources path type); - # As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses # gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++ stdenv = @@ -111,36 +98,22 @@ else pkgs.llvmPackages.stdenv; - extraBuildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [ - # Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156 - pkgs.libiconv - pkgs.darwin.apple_sdk.frameworks.Security - ]; - - sharedArgs = { + # Combine the environment and other configuration needed for crane to build our Rust packages + commonArgs = environment // { + pname = "noir"; # x-release-please-start-version version = "0.6.0"; # x-release-please-end - src = pkgs.lib.cleanSourceWith { - src = craneLib.path ./.; - filter = sourceFilter; - }; - - # TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell. - cargoClippyExtraArgs = "--all-targets -- -D warnings"; - - # TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell. - cargoTestExtraArgs = "--workspace"; - }; - - # Combine the environment and other configuration needed for crane to build our Rust packages - nativeArgs = nativeEnvironment // sharedArgs // { - pname = "noir-native"; - # Use our custom stdenv to build and test our Rust project inherit stdenv; + src = ./.; + + # Running checks don't do much more than compiling itself and increase + # the build time by a lot, so we disable them throughout all our flakes + doCheck = false; + nativeBuildInputs = [ # This provides the pkg-config tool to find barretenberg & other native libraries pkgs.pkg-config @@ -151,21 +124,18 @@ buildInputs = [ pkgs.llvmPackages.openmp pkgs.barretenberg - ] ++ extraBuildInputs; - }; - - # Combine the environment and other configuration needed for crane to build with the wasm feature - wasmArgs = wasmEnvironment // sharedArgs // { - pname = "noir-wasm"; - - # We disable the default "plonk_bn254" feature and enable the "plonk_bn254_wasm" feature - cargoExtraArgs = "--no-default-features --features='plonk_bn254_wasm'"; + ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ + # Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156 + pkgs.libiconv + pkgs.darwin.apple_sdk.frameworks.Security + ]; - buildInputs = [ ] ++ extraBuildInputs; + inherit GIT_COMMIT; + inherit GIT_DIRTY; }; # The `port` is parameterized to support parallel test runs without colliding static servers - testArgs = port: testEnvironment // { + testArgs = port: { # We provide `barretenberg-transcript00` from the overlay to the tests as a URL hosted via a static server # This is necessary because the Nix sandbox has no network access and downloading during tests would fail TRANSCRIPT_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf pkgs.barretenberg-transcript00}"; @@ -190,52 +160,36 @@ }; # Build *just* the cargo dependencies, so we can reuse all of that work between runs - native-cargo-artifacts = craneLib.buildDepsOnly nativeArgs; - wasm-cargo-artifacts = craneLib.buildDepsOnly wasmArgs; - - noir-native = craneLib.buildPackage (nativeArgs // { - inherit GIT_COMMIT GIT_DIRTY; + cargoArtifacts = craneLib.buildDepsOnly commonArgs; - cargoArtifacts = native-cargo-artifacts; - - # We don't want to run checks or tests when just building the project - doCheck = false; - }); - - noir-wasm = craneLib.buildPackage (wasmArgs // { - inherit GIT_COMMIT GIT_DIRTY; - - cargoArtifacts = wasm-cargo-artifacts; - - # We don't want to run checks or tests when just building the project - doCheck = false; + noir = craneLib.buildPackage (commonArgs // { + inherit cargoArtifacts; }); in rec { checks = { - cargo-clippy = craneLib.cargoClippy (nativeArgs // { - inherit GIT_COMMIT GIT_DIRTY; + cargo-clippy = craneLib.cargoClippy (commonArgs // { + inherit cargoArtifacts; - cargoArtifacts = native-cargo-artifacts; + # TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell. + cargoClippyExtraArgs = "--all-targets -- -D warnings"; }); - cargo-test = craneLib.cargoTest (nativeArgs // (testArgs 8000) // { - inherit GIT_COMMIT GIT_DIRTY; + cargo-test = craneLib.cargoTest (commonArgs // (testArgs 8000) // { + inherit cargoArtifacts; + + # TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell. + cargoTestExtraArgs = "--workspace"; - cargoArtifacts = native-cargo-artifacts; + # It's unclear why doCheck needs to be enabled for tests to run but not clippy + doCheck = true; }); }; - packages = { - default = noir-native; + packages.default = noir; - inherit noir-native; - inherit noir-wasm; - - # We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI - inherit native-cargo-artifacts; - inherit wasm-cargo-artifacts; - }; + # We expose the `cargo-artifacts` derivation so we can cache our cargo dependencies in CI + packages.cargo-artifacts = cargoArtifacts; # TODO(#1197): Look into installable apps with Nix flakes # apps.default = flake-utils.lib.mkApp { drv = nargo; }; @@ -243,7 +197,7 @@ # Setup the environment to match the stdenv from `nix build` & `nix flake check`, and # combine it with the environment settings, the inputs from our checks derivations, # and extra tooling via `nativeBuildInputs` - devShells.default = pkgs.mkShell.override { inherit stdenv; } (nativeEnvironment // wasmEnvironment // testEnvironment // { + devShells.default = pkgs.mkShell.override { inherit stdenv; } (environment // { inputsFrom = builtins.attrValues checks; nativeBuildInputs = with pkgs; [