From 74b83c4e8e56e78d027275f4bd021b1f985468ae Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 22:26:52 -0400 Subject: [PATCH 1/3] Try to substitute builtins.fetch* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit builtins.fetch* aren’t technically derivations, so they never got substituted. To work around this, we can try to substitute this store path during evaluation. This can happen any time a hash is provided. Updated to be closer to how the flakes branch works. --- src/libexpr/primops/fetchTree.cc | 14 ++++++++++++++ src/libfetchers/fetchers.cc | 18 ++++++++++++++++++ src/libfetchers/fetchers.hh | 3 +++ 3 files changed, 35 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9be93710ad3..6725ab22e4f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -134,6 +134,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + // try to substitute if we can + if (settings.useSubstitutes && expectedHash) { + auto substitutableStorePath = fetchers::trySubstitute(state.store, + unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, *expectedHash, name); + if (substitutableStorePath) { + auto substitutablePath = state.store->toRealPath(*substitutableStorePath); + if (state.allowedPaths) + state.allowedPaths->insert(substitutablePath); + + mkString(v, substitutablePath, PathSet({substitutablePath})); + return; + } + } + auto storePath = unpack ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9174c3de467..ec0510652f4 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -72,4 +72,22 @@ std::pair> Input::fetchTree(ref store) return {std::move(tree), input}; } +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name) +{ + auto substitutablePath = store->makeFixedOutputPath(ingestionMethod, hash, name); + + try { + store->ensurePath(substitutablePath); + + debug("using substituted path '%s'", store->printStorePath(substitutablePath)); + + return substitutablePath; + } catch (Error & e) { + debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what()); + } + + return std::nullopt; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae6781..2eeaed2f287 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -100,4 +100,7 @@ Tree downloadTarball( const std::string & name, bool immutable); +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name); + } From 5fa33d70a85f0a29e218d38dd6ced6c4086332de Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 22 Jun 2020 18:08:24 -0400 Subject: [PATCH 2/3] Support substituting in fetchGit --- src/libfetchers/git.cc | 20 +++++++++++++++++++- tests/git.sh | 9 +++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 85902126dbf..49587cf461b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -106,6 +106,24 @@ struct GitInput : Input auto ingestionMethod = treeHash ? FileIngestionMethod::Git : FileIngestionMethod::Recursive; + // try to substitute + if (settings.useSubstitutes && treeHash && !submodules) { + auto storePath = fetchers::trySubstitute(store, ingestionMethod, *treeHash, name); + if (storePath) { + return { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = std::nullopt, + .lastModified = 0, + }, + }, + input + }; + } + } + std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; if (submodules) cacheType += "-submodules"; @@ -385,7 +403,7 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, ingestionMethod, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); // verify treeHash is what we actually obtained in the nix store if (input->treeHash) { diff --git a/tests/git.sh b/tests/git.sh index bdc98844a0d..cd9979ada29 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -1,6 +1,7 @@ source common.sh clearStore +clearCache try () { hash=$(nix hash-git --base16 --type sha1 $TEST_ROOT/hash-path) @@ -40,8 +41,6 @@ test "$hash3" = "sha256:1i2x80840igikhbyy7nqf08ymx3a6n83x1fzyrxvddf0sdl5nqvp" if [[ -n $(type -p git) ]]; then repo=$TEST_ROOT/git - export _NIX_FORCE_HTTP=1 - rm -rf $repo $TEST_HOME/.cache/nix git init $repo @@ -64,6 +63,12 @@ if [[ -n $(type -p git) ]]; then # Submodules cause error. (! nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; submodules = true; }).outPath") + + # Check that we can substitute it from other places. + nix copy --to file://$cacheDir $path + nix-store --delete $path + path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir) + [ $path2 = $path ] else echo "Git not installed; skipping Git tests" fi From 4074142e688ee156739461343a460b838ea48e92 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 13:34:04 -0400 Subject: [PATCH 3/3] Override substitute option in test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is disabled automatically when we’re offline. --- tests/git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/git.sh b/tests/git.sh index cd9979ada29..85ce99f6c40 100644 --- a/tests/git.sh +++ b/tests/git.sh @@ -67,7 +67,7 @@ if [[ -n $(type -p git) ]]; then # Check that we can substitute it from other places. nix copy --to file://$cacheDir $path nix-store --delete $path - path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir) + path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters file://$cacheDir --option substitute true) [ $path2 = $path ] else echo "Git not installed; skipping Git tests"