Skip to content

Commit

Permalink
Revert "chore(nix): Fix dependency caching (#1421)"
Browse files Browse the repository at this point in the history 
This reverts commit 9081ec9.
  • Loading branch information
@kevaundray
kevaundray committed Jun 1, 2023
1 parent 944eba3 commit 2a8a40c
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 89 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,4 @@ jobs:
- name: Export cache from nix store
if: steps.nix-store-cache.outputs.cache-hit != 'true'
run: |
nix copy --to "file:///tmp/nix-cache?compression=zstd&parallel-compression=true" .#native-cargo-artifacts
nix copy --to "file:///tmp/nix-cache?compression=zstd&parallel-compression=true" .#cargo-artifacts
130 changes: 42 additions & 88 deletions flake.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,41 +68,28 @@

craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;

sharedEnvironment = {
# We enable backtraces on any failure for help with debugging
RUST_BACKTRACE = "1";
};

nativeEnvironment = sharedEnvironment // {
environment = {
# rust-bindgen needs to know the location of libclang
LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
};

wasmEnvironment = sharedEnvironment // {
# We set the environment variable because barretenberg must be compiled in a special way for wasm
BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin";
};

testEnvironment = sharedEnvironment // {
# Barretenberg fails if tests are run on multiple threads, so we set the test thread
# count to 1 throughout the entire project
#
# Note: Setting this allows for consistent behavior across build and shells, but is mostly
# hidden from the developer - i.e. when they see the command being run via `nix flake check`
RUST_TEST_THREADS = "1";

# We enable backtraces on any failure for help with debugging
RUST_BACKTRACE = "1";

# We set the environment variable because barretenberg must be compiled in a special way for wasm
BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin";
};

# The `self.rev` property is only available when the working tree is not dirty
GIT_COMMIT = if (self ? rev) then self.rev else "unknown";
GIT_DIRTY = if (self ? rev) then "false" else "true";

# We use `.nr` and `.toml` files in tests so we need to create a special source
# filter to include those files in addition to usual rust/cargo source files
noirFilter = path: _type: builtins.match ".*nr$" path != null;
tomlFilter = path: _type: builtins.match ".*toml$" path != null;
sourceFilter = path: type:
(noirFilter path type) || (tomlFilter path type) || (craneLib.filterCargoSources path type);

# As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses
# gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++
stdenv =
Expand All @@ -111,36 +98,22 @@
else
pkgs.llvmPackages.stdenv;

extraBuildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [
# Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156
pkgs.libiconv
pkgs.darwin.apple_sdk.frameworks.Security
];

sharedArgs = {
# Combine the environment and other configuration needed for crane to build our Rust packages
commonArgs = environment // {
pname = "noir";
# x-release-please-start-version
version = "0.6.0";
# x-release-please-end

src = pkgs.lib.cleanSourceWith {
src = craneLib.path ./.;
filter = sourceFilter;
};

# TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell.
cargoClippyExtraArgs = "--all-targets -- -D warnings";

# TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell.
cargoTestExtraArgs = "--workspace";
};

# Combine the environment and other configuration needed for crane to build our Rust packages
nativeArgs = nativeEnvironment // sharedArgs // {
pname = "noir-native";

# Use our custom stdenv to build and test our Rust project
inherit stdenv;

src = ./.;

# Running checks don't do much more than compiling itself and increase
# the build time by a lot, so we disable them throughout all our flakes
doCheck = false;

nativeBuildInputs = [
# This provides the pkg-config tool to find barretenberg & other native libraries
pkgs.pkg-config
Expand All @@ -151,21 +124,18 @@
buildInputs = [
pkgs.llvmPackages.openmp
pkgs.barretenberg
] ++ extraBuildInputs;
};

# Combine the environment and other configuration needed for crane to build with the wasm feature
wasmArgs = wasmEnvironment // sharedArgs // {
pname = "noir-wasm";

# We disable the default "plonk_bn254" feature and enable the "plonk_bn254_wasm" feature
cargoExtraArgs = "--no-default-features --features='plonk_bn254_wasm'";
] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
# Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156
pkgs.libiconv
pkgs.darwin.apple_sdk.frameworks.Security
];

buildInputs = [ ] ++ extraBuildInputs;
inherit GIT_COMMIT;
inherit GIT_DIRTY;
};

# The `port` is parameterized to support parallel test runs without colliding static servers
testArgs = port: testEnvironment // {
testArgs = port: {
# We provide `barretenberg-transcript00` from the overlay to the tests as a URL hosted via a static server
# This is necessary because the Nix sandbox has no network access and downloading during tests would fail
TRANSCRIPT_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf pkgs.barretenberg-transcript00}";
Expand All @@ -190,60 +160,44 @@
};

# Build *just* the cargo dependencies, so we can reuse all of that work between runs
native-cargo-artifacts = craneLib.buildDepsOnly nativeArgs;
wasm-cargo-artifacts = craneLib.buildDepsOnly wasmArgs;

noir-native = craneLib.buildPackage (nativeArgs // {
inherit GIT_COMMIT GIT_DIRTY;
cargoArtifacts = craneLib.buildDepsOnly commonArgs;

cargoArtifacts = native-cargo-artifacts;

# We don't want to run checks or tests when just building the project
doCheck = false;
});

noir-wasm = craneLib.buildPackage (wasmArgs // {
inherit GIT_COMMIT GIT_DIRTY;

cargoArtifacts = wasm-cargo-artifacts;

# We don't want to run checks or tests when just building the project
doCheck = false;
noir = craneLib.buildPackage (commonArgs // {
inherit cargoArtifacts;
});
in
rec {
checks = {
cargo-clippy = craneLib.cargoClippy (nativeArgs // {
inherit GIT_COMMIT GIT_DIRTY;
cargo-clippy = craneLib.cargoClippy (commonArgs // {
inherit cargoArtifacts;

cargoArtifacts = native-cargo-artifacts;
# TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell.
cargoClippyExtraArgs = "--all-targets -- -D warnings";
});

cargo-test = craneLib.cargoTest (nativeArgs // (testArgs 8000) // {
inherit GIT_COMMIT GIT_DIRTY;
cargo-test = craneLib.cargoTest (commonArgs // (testArgs 8000) // {
inherit cargoArtifacts;

# TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell.
cargoTestExtraArgs = "--workspace";

cargoArtifacts = native-cargo-artifacts;
# It's unclear why doCheck needs to be enabled for tests to run but not clippy
doCheck = true;
});
};

packages = {
default = noir-native;
packages.default = noir;

inherit noir-native;
inherit noir-wasm;

# We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI
inherit native-cargo-artifacts;
inherit wasm-cargo-artifacts;
};
# We expose the `cargo-artifacts` derivation so we can cache our cargo dependencies in CI
packages.cargo-artifacts = cargoArtifacts;

# TODO(#1197): Look into installable apps with Nix flakes
# apps.default = flake-utils.lib.mkApp { drv = nargo; };

# Setup the environment to match the stdenv from `nix build` & `nix flake check`, and
# combine it with the environment settings, the inputs from our checks derivations,
# and extra tooling via `nativeBuildInputs`
devShells.default = pkgs.mkShell.override { inherit stdenv; } (nativeEnvironment // wasmEnvironment // testEnvironment // {
devShells.default = pkgs.mkShell.override { inherit stdenv; } (environment // {
inputsFrom = builtins.attrValues checks;

nativeBuildInputs = with pkgs; [
Expand Down

0 comments on commit 2a8a40c

Please sign in to comment.