From ad1588e985140c03110cf136c7ee527722b4aa8c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 12:38:06 -0400 Subject: [PATCH 01/15] Add ContentAddress to store-api path methods --- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/binary-cache-store.hh | 6 +++--- src/libstore/build.cc | 6 +++--- src/libstore/ipfs-binary-cache-store.cc | 8 ++++---- src/libstore/legacy-ssh-store.cc | 5 +++-- src/libstore/local-fs-store.cc | 2 +- src/libstore/local-store.cc | 6 +++--- src/libstore/local-store.hh | 4 ++-- src/libstore/remote-store.cc | 4 ++-- src/libstore/remote-store.hh | 4 ++-- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/ssh-store.cc | 4 ++-- src/libstore/store-api.cc | 14 +++++++------- src/libstore/store-api.hh | 14 +++++++------- 14 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e940ceff132..b1d103bf6e7 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -263,7 +263,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) +bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath, std::optional ca) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even @@ -271,7 +271,7 @@ bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink, std::optional ca) { auto info = queryPathInfo(storePath).cast(); @@ -298,7 +298,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, - Callback> callback) noexcept + Callback> callback, std::optional ca) noexcept { auto uri = getUri(); auto storePathS = printStorePath(storePath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 52ef8aa7ac2..1d84f1286f6 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -66,10 +66,10 @@ private: public: - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(const StorePath & path, std::optional ca) override; void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; + Callback> callback, std::optional ca) noexcept override; std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } @@ -85,7 +85,7 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink, std::optional ca) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 697c3fcfff3..b84cf86d5bc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2729,7 +2729,7 @@ struct RestrictedStore : public LocalFSStore } void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override + Callback> callback, std::optional ca) noexcept override { if (goal.isAllowed(path)) { try { @@ -2785,11 +2785,11 @@ struct RestrictedStore : public LocalFSStore return path; } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink, std::optional ca) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + LocalFSStore::narFromPath(path, sink, ca); } void ensurePath(const StorePath & path) override diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index fa2cb4d2c29..bc3440771fc 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -487,7 +487,7 @@ class IPFSBinaryCacheStore : public Store stats.narInfoWrite++; } - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(const StorePath & storePath, std::optional ca) override { auto json = getIpfsDag(getIpfsPath()); if (!json.contains("nar")) @@ -495,9 +495,9 @@ class IPFSBinaryCacheStore : public Store return json["nar"].contains(storePath.to_string()); } - void narFromPath(const StorePath & storePath, Sink & sink) override + void narFromPath(const StorePath & storePath, Sink & sink, std::optional ca) override { - auto info = queryPathInfo(storePath).cast(); + auto info = queryPathInfo(storePath, ca).cast(); uint64_t narSize = 0; @@ -522,7 +522,7 @@ class IPFSBinaryCacheStore : public Store } void queryPathInfoUncached(const StorePath & storePath, - Callback> callback) noexcept override + Callback> callback, std::optional ca) noexcept override { // TODO: properly use callbacks diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593be..18728684666 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,8 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override + Callback> callback, + std::optional ca) noexcept override { try { auto conn(connections->get()); @@ -182,7 +183,7 @@ struct LegacySSHStore : public Store throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink, std::optional ca) override { auto conn(connections->get()); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index dd96d2578b4..1159a725801 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -77,7 +77,7 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePath & path, Sink & sink, std::optional ca) { if (!isValidPath(path)) throw Error("path '%s' is not valid", printStorePath(path)); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b2361c7b93d..ac13dbff9db 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -631,7 +631,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept + Callback> callback, std::optional ca) noexcept { try { auto info = std::make_shared(path); @@ -711,7 +711,7 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) } -bool LocalStore::isValidPathUncached(const StorePath & path) +bool LocalStore::isValidPathUncached(const StorePath & path, std::optional ca) { return retrySQLite([&]() { auto state(_state.lock()); @@ -862,7 +862,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); try { - auto info = sub->queryPathInfo(subPath); + auto info = sub->queryPathInfo(subPath, path.second); if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) continue; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ab9fbfbe40c..46b0b160d3b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -119,7 +119,7 @@ public: std::string getUri() override; - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(const StorePath & path, std::optional ca) override; StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; @@ -127,7 +127,7 @@ public: StorePathSet queryAllValidPaths() override; void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; + Callback> callback, std::optional ca) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 94eb84156f4..f1b3807bbf6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -254,7 +254,7 @@ ConnectionHandle RemoteStore::getConnection() } -bool RemoteStore::isValidPathUncached(const StorePath & path) +bool RemoteStore::isValidPathUncached(const StorePath & path, std::optional ca) { auto conn(getConnection()); conn->to << wopIsValidPath << printStorePath(path); @@ -356,7 +356,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S void RemoteStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept + Callback> callback, std::optional ca) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 07ed2b51a1d..9659abd2dbd 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,7 +35,7 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(const StorePath & path, std::optional ca) override; StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; @@ -43,7 +43,7 @@ public: StorePathSet queryAllValidPaths() override; void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; + Callback> callback, std::optional ca) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 427dd48ce36..84522319237 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -227,7 +227,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(const StorePath & storePath, std::optional ca) override { try { queryPathInfo(storePath); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596c7..2af608e5352 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -40,7 +40,7 @@ class SSHStore : public RemoteStore bool sameMachine() override { return false; } - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink, std::optional ca) override; ref getFSAccessor() override; @@ -68,7 +68,7 @@ class SSHStore : public RemoteStore }; }; -void SSHStore::narFromPath(const StorePath & path, Sink & sink) +void SSHStore::narFromPath(const StorePath & path, Sink & sink, std::optional ca) { auto conn(connections->get()); conn->to << wopNarFromPath << printStorePath(path); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a459fe429b4..83e647876d6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -274,7 +274,7 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -bool Store::isValidPath(const StorePath & storePath) +bool Store::isValidPath(const StorePath & storePath, std::optional ca) { std::string hashPart(storePath.hashPart()); @@ -298,7 +298,7 @@ bool Store::isValidPath(const StorePath & storePath) } } - bool valid = isValidPathUncached(storePath); + bool valid = isValidPathUncached(storePath, ca); if (diskCache && !valid) // FIXME: handle valid = true case. @@ -310,7 +310,7 @@ bool Store::isValidPath(const StorePath & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const StorePath & path) +bool Store::isValidPathUncached(const StorePath & path, std::optional ca) { try { queryPathInfo(path); @@ -321,7 +321,7 @@ bool Store::isValidPathUncached(const StorePath & path) } -ref Store::queryPathInfo(const StorePath & storePath) +ref Store::queryPathInfo(const StorePath & storePath, std::optional ca) { std::promise> promise; @@ -332,14 +332,14 @@ ref Store::queryPathInfo(const StorePath & storePath) } catch (...) { promise.set_exception(std::current_exception()); } - }}); + }}, ca); return promise.get_future().get(); } void Store::queryPathInfo(const StorePath & storePath, - Callback> callback) noexcept + Callback> callback, std::optional ca) noexcept { std::string hashPart; @@ -397,7 +397,7 @@ void Store::queryPathInfo(const StorePath & storePath, (*callbackPtr)(ref(info)); } catch (...) { callbackPtr->rethrow(); } - }}); + }}, ca); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 624aad73f92..4ad45acbdbc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -374,11 +374,11 @@ public: const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const StorePath & path); + bool isValidPath(const StorePath & path, std::optional ca = std::nullopt); protected: - virtual bool isValidPathUncached(const StorePath & path); + virtual bool isValidPathUncached(const StorePath & path, std::optional ca = std::nullopt); public: @@ -397,16 +397,16 @@ public: /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref queryPathInfo(const StorePath & path); + ref queryPathInfo(const StorePath & path, std::optional ca = std::nullopt); /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const StorePath & path, - Callback> callback) noexcept; + Callback> callback, std::optional ca = std::nullopt) noexcept; protected: virtual void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept = 0; + Callback> callback, std::optional ca = std::nullopt) noexcept = 0; public: @@ -464,7 +464,7 @@ public: const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink, std::optional ca = std::nullopt) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -711,7 +711,7 @@ public: LocalFSStore(const Params & params); - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink, std::optional) override; ref getFSAccessor() override; /* Register a permanent GC root. */ From 9c1fdf2fcb1694720f210d3e134e17000269cc94 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 12:39:12 -0400 Subject: [PATCH 02/15] Handle CA value in ipfs-binary-cache-store When this is provided, we can look up a f01781114 hash in ipfs. --- src/libstore/ipfs-binary-cache-store.cc | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index bc3440771fc..50c2d4ec5eb 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -170,6 +170,27 @@ class IPFSBinaryCacheStore : public Store } } + std::optional ipfsBlockStat(std::string ipfsPath) + { + auto uri = daemonUri + "/api/v0/block/stat?arg=" + getFileTransfer()->urlEncode(ipfsPath); + + FileTransferRequest request(uri); + request.post = true; + request.tries = 1; + try { + auto res = getFileTransfer()->download(request); + auto json = nlohmann::json::parse(*res.data); + + if (json.find("Size") != json.end()) + return (uint64_t) json["Size"]; + } catch (FileTransferError & e) { + // probably should verify this is a not found error but + // ipfs gives us a 500 + } + + return std::nullopt; + } + bool fileExists(const std::string & path) { return ipfsObjectExists(getIpfsPath() + "/" + path); @@ -426,6 +447,23 @@ class IPFSBinaryCacheStore : public Store } } + // ::= + // f = base16 + // cid-version = 01 + // multicodec-packed-content-type = 1114 + std::optional getCidFromCA(ContentAddress ca) + { + if (std::holds_alternative(ca)) { + auto ca_ = std::get(ca); + if (ca_.method == FileIngestionMethod::Git) { + assert(ca_.hash.type == htSHA1); + return "f01781114" + ca_.hash.to_string(Base16, false); + } + } + + return std::nullopt; + } + public: void addToStore(const ValidPathInfo & info, Source & narSource, @@ -489,6 +527,12 @@ class IPFSBinaryCacheStore : public Store bool isValidPathUncached(const StorePath & storePath, std::optional ca) override { + if (ca) { + auto cid = getCidFromCA(*ca); + if (cid && ipfsBlockStat("/ipfs/" + *cid)) + return true; + } + auto json = getIpfsDag(getIpfsPath()); if (!json.contains("nar")) return false; @@ -534,6 +578,22 @@ class IPFSBinaryCacheStore : public Store fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); + if (ca) { + auto cid = getCidFromCA(*ca); + if (cid) { + auto size = ipfsBlockStat("/ipfs/" + *cid); + if (size) { + assert(storePath == makeFixedOutputPathFromCA(storePath.name(), *ca)); + NarInfo narInfo { storePath }; + narInfo.ca = ca; + narInfo.url = "ipfs://" + *cid; + narInfo.narHash = std::get(*ca).hash; + narInfo.narSize = *size; + return; + } + } + } + auto json = getIpfsDag(getIpfsPath()); if (!json.contains("nar") || !json["nar"].contains(storePath.to_string())) From e7681e4576c7f7ac07d4e8e4ed02f75b65d86f94 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 15:07:17 -0400 Subject: [PATCH 03/15] Use NARs for transfering git paths This makes things simpler and avoids any incompatiblities with the daemon. We always just send thing the NAR format in dumps. This should map one to one with Git objects. --- src/libstore/daemon.cc | 16 +++--------- src/libstore/local-store.cc | 48 +++++++++++++++--------------------- src/libstore/remote-store.cc | 5 +--- src/libutil/git.cc | 1 + 4 files changed, 25 insertions(+), 45 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 520849842b7..9585959de12 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -387,24 +387,14 @@ static void performOp(TunnelLogger * logger, ref store, TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - switch (method) { - case FileIngestionMethod::Recursive: { + if (method == FileIngestionMethod::Flat) + parseDump(savedRegular, from); + else { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ ParseSink sink; /* null sink; just parse the NAR */ parseDump(sink, savedNAR); - break; - } - case FileIngestionMethod::Git: { - ParseSink sink; - parseGit(sink, savedNAR, store->storeDir, store->storeDir); - break; - } - case FileIngestionMethod::Flat: { - parseDump(savedRegular, from); - break; - } } logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ac13dbff9db..a8ea6430d11 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1051,6 +1051,15 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam Hash h = hashString(hashAlgo, dump); + // ugh... we need to calculate the hash just to get what path we + // have. dump is still just a NAR, so we make our own dump. + if (method == FileIngestionMethod::Git) { + AutoDelete tmpDir(createTempDir(), true); + StringSource source(dump); + restorePath((Path) tmpDir + "/tmp", source); + h = dumpGitHash(hashAlgo, (Path) tmpDir + "/tmp"); + } + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -1071,20 +1080,11 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - switch (method) { - case FileIngestionMethod::Flat: + if (method == FileIngestionMethod::Flat) writeFile(realPath, dump); - break; - case FileIngestionMethod::Recursive: { + else { StringSource source(dump); restorePath(realPath, source); - break; - } - case FileIngestionMethod::Git: { - StringSource source(dump); - restoreGit(realPath, source, realStoreDir, storeDir); - break; - } } canonicalisePathMetaData(realPath, -1); @@ -1124,16 +1124,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) throw Error("git ingestion must use sha1 hash"); - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - switch (method) { - case FileIngestionMethod::Recursive: { - dumpPath(srcPath, sink, filter); - break; - } - case FileIngestionMethod::Git: { + if (method == FileIngestionMethod::Git) { // recursively add to store if path is a directory struct stat st; if (lstat(srcPath.c_str(), &st)) @@ -1141,15 +1132,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (S_ISDIR(st.st_mode)) for (auto & i : readDirectory(srcPath)) addToStore("git", srcPath + "/" + i.name, method, hashAlgo, filter, repair); - - dumpGit(hashAlgo, srcPath, sink, filter); - break; } - case FileIngestionMethod::Flat: { + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (method == FileIngestionMethod::Flat) sink.s = make_ref(readFile(srcPath)); - break; - } - } + else + dumpPath(srcPath, sink, filter); return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f1b3807bbf6..4b10e530ef8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -512,10 +512,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (method == FileIngestionMethod::Git) - dumpGit(hashAlgo, srcPath, conn->to, filter); - else - dumpPath(srcPath, conn->to, filter); + dumpPath(srcPath, conn->to, filter); } conn->to.warn = false; conn.processStderr(); diff --git a/src/libutil/git.cc b/src/libutil/git.cc index b0356c419b9..364d28e468a 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -217,6 +217,7 @@ static GitMode dumpGitInternal(HashType ht, const Path & path, Sink & sink, Path static std::pair dumpGitHashInternal(HashType ht, const Path & path, PathFilter & filter) { + assert(ht == htSHA1); auto hashSink = new HashSink(ht); auto perm = dumpGitInternal(ht, path, *hashSink, filter); auto hash = hashSink->finish().first; From ee5a2dabe8ad17fcfa058d11bbd225901d3d7728 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 17:47:39 -0400 Subject: [PATCH 04/15] Add ca to more store api parameters ensurePath needs to know ca as well --- src/libfetchers/fetchers.cc | 6 +++++- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build.cc | 16 ++++++++-------- src/libstore/ipfs-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 9 +++++---- src/libstore/store-api.hh | 6 ++++-- 10 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index ec0510652f4..1be05f48045 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -78,7 +78,11 @@ std::optional trySubstitute(ref store, FileIngestionMethod ing auto substitutablePath = store->makeFixedOutputPath(ingestionMethod, hash, name); try { - store->ensurePath(substitutablePath); + auto ca = FixedOutputHash { + .method = ingestionMethod, + .hash = hash + }; + store->ensurePath(substitutablePath, ca); debug("using substituted path '%s'", store->printStorePath(substitutablePath)); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 1d84f1286f6..6649191241b 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -91,7 +91,7 @@ public: BuildMode buildMode) override { unsupported("buildDerivation"); } - void ensurePath(const StorePath & path) override + void ensurePath(const StorePath & path, std::optional ca) override { unsupported("ensurePath"); } ref getFSAccessor() override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b84cf86d5bc..ceea131cc0e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore if (goal.isAllowed(path)) { try { /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); + auto info = std::make_shared(*next->queryPathInfo(path, ca)); info->deriver.reset(); info->registrationTime = 0; info->ultimate = false; @@ -2792,7 +2792,7 @@ struct RestrictedStore : public LocalFSStore LocalFSStore::narFromPath(path, sink, ca); } - void ensurePath(const StorePath & path) override + void ensurePath(const StorePath & path, std::optional ca) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); @@ -4422,7 +4422,7 @@ void SubstitutionGoal::tryNext() try { // FIXME: make async - info = sub->queryPathInfo(subPath); + info = sub->queryPathInfo(subPath, ca); } catch (InvalidPath &) { tryNext(); return; @@ -4550,7 +4550,7 @@ void SubstitutionGoal::tryToRun() } copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs, ca); promise.set_value(); } catch (...) { @@ -5126,15 +5126,15 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe } -void LocalStore::ensurePath(const StorePath & path) +void LocalStore::ensurePath(const StorePath & path, std::optional ca) { /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; + if (isValidPath(path, ca)) return; - primeCache(*this, {{path}}); + // primeCache(*this, {{path}}); Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); + GoalPtr goal = worker.makeSubstitutionGoal(path, NoRepair, ca); Goals goals = {goal}; worker.run(goals); diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 50c2d4ec5eb..419897a2f7f 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -711,7 +711,7 @@ class IPFSBinaryCacheStore : public Store BuildMode buildMode) override { unsupported("buildDerivation"); } - void ensurePath(const StorePath & path) override + void ensurePath(const StorePath & path, std::optional ca) override { unsupported("ensurePath"); } std::optional queryPathFromHashPart(const std::string & hashPart) override diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 18728684666..50f54ad1b20 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -236,7 +236,7 @@ struct LegacySSHStore : public Store return status; } - void ensurePath(const StorePath & path) override + void ensurePath(const StorePath & path, std::optional ca) override { unsupported("ensurePath"); } void computeFSClosure(const StorePathSet & paths, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 46b0b160d3b..7791feee7df 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -167,7 +167,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(const StorePath & path, std::optional ca) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4b10e530ef8..4fccf47a122 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -581,7 +581,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } -void RemoteStore::ensurePath(const StorePath & path) +void RemoteStore::ensurePath(const StorePath & path, std::optional ca) { auto conn(getConnection()); conn->to << wopEnsurePath << printStorePath(path); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9659abd2dbd..c4a30418595 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -74,7 +74,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(const StorePath & path, std::optional ca) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83e647876d6..836bd3ea66e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -595,7 +595,8 @@ void Store::buildPaths(const std::vector & paths, BuildMod void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) + const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs, + std::optional ca) { auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); @@ -609,7 +610,7 @@ void copyStorePath(ref srcStore, ref dstStore, {srcStore->printStorePath(storePath), srcUri, dstUri}); PushActivity pact(act.id); - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore->queryPathInfo(storePath, ca); uint64_t total = 0; @@ -624,7 +625,7 @@ void copyStorePath(ref srcStore, ref dstStore, if (!info->narHash) { StringSink sink; - srcStore->narFromPath({storePath}, sink); + srcStore->narFromPath({storePath}, sink, ca); auto info2 = make_ref(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); @@ -648,7 +649,7 @@ void copyStorePath(ref srcStore, ref dstStore, total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath(storePath, wrapperSink); + srcStore->narFromPath(storePath, wrapperSink, ca); }, [&]() { throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4ad45acbdbc..2f38f98a69b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -487,7 +487,8 @@ public: /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const StorePath & path) = 0; + virtual void ensurePath(const StorePath & path, + std::optional ca = std::nullopt) = 0; /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ @@ -732,7 +733,8 @@ public: /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::optional ca = std::nullopt); /* Copy store paths from one store to another. The paths may be copied From 70b1584fb856971b991402686e1a10cd5c57bb12 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 17:48:46 -0400 Subject: [PATCH 05/15] Use /api/v0/block/get instead of cat these appear to be very similar, with the main exception that /block/get works on non-UnixFS stuff. --- src/libstore/ipfs-binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 419897a2f7f..0cb5476c085 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -361,7 +361,7 @@ class IPFSBinaryCacheStore : public Store void getIpfsObject(const std::string & ipfsPath, Callback> callback) noexcept { - auto uri = daemonUri + "/api/v0/cat?arg=" + getFileTransfer()->urlEncode(ipfsPath); + auto uri = daemonUri + "/api/v0/block/get?arg=" + getFileTransfer()->urlEncode(ipfsPath); FileTransferRequest request(uri); request.post = true; From 1639f71e2f37df00e642d7559dee509848426fcd Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 17:49:55 -0400 Subject: [PATCH 06/15] Fix ipfs stream handling need to correctly handle nar info data --- src/libstore/ipfs-binary-cache-store.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 0cb5476c085..dcc28a82084 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -589,6 +589,8 @@ class IPFSBinaryCacheStore : public Store narInfo.url = "ipfs://" + *cid; narInfo.narHash = std::get(*ca).hash; narInfo.narSize = *size; + (*callbackPtr)((std::shared_ptr) + std::make_shared(narInfo)); return; } } @@ -657,13 +659,13 @@ class IPFSBinaryCacheStore : public Store small files. */ StringSink sink; Hash h; - if (method == FileIngestionMethod::Recursive) { - dumpPath(srcPath, sink, filter); - h = hashString(hashAlgo, *sink.s); - } else { + if (method == FileIngestionMethod::Flat) { auto s = readFile(srcPath); dumpString(s, sink); h = hashString(hashAlgo, s); + } else { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); } ValidPathInfo info(makeFixedOutputPath(method, h, name)); From ed09319a380c99c8c9f810fea7a271e63f320911 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 17:50:33 -0400 Subject: [PATCH 07/15] Upload git objects directly to ipfs --- src/libstore/ipfs-binary-cache-store.cc | 49 ++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index dcc28a82084..6bf8a1e0835 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -6,6 +6,7 @@ #include "nar-info-disk-cache.hh" #include "archive.hh" #include "compression.hh" +#include "git.hh" #include "names.hh" namespace nix { @@ -464,16 +465,60 @@ class IPFSBinaryCacheStore : public Store return std::nullopt; } + void putIpfsGitBlock(std::string s) + { + auto uri = daemonUri + "/api/v0/block/put?format=git-raw&mhtype=sha1"; + + auto req = FileTransferRequest(uri); + req.data = std::make_shared(s); + req.post = true; + req.tries = 1; + getFileTransfer()->upload(req); + } + + void addGit(Path path) + { + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting attributes of path '%1%'", path); + + if (S_ISREG(st.st_mode)) { + StringSink sink; + dumpGitBlob(path, st, sink); + putIpfsGitBlock(*sink.s); + } else if (S_ISDIR(st.st_mode)) { + for (auto & i : readDirectory(path)) + addGit(path + "/" + i.name); + StringSink sink; + dumpGit(htSHA1, path, sink); + putIpfsGitBlock(*sink.s); + } else throw Error("file '%1%' has an unsupported type", path); + } + public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override { - // FIXME: See if we can use the original source to reduce memory usage. - auto nar = make_ref(narSource.drain()); if (!repair && isValidPath(info.path)) return; + // Note this doesn’t require a IPFS store, it just goes in the + // global namespace. + if (info.ca && std::holds_alternative(*info.ca)) { + auto ca_ = std::get(*info.ca); + if (ca_.method == FileIngestionMethod::Git) { + AutoDelete tmpDir(createTempDir(), true); + TeeSource savedNAR(narSource); + restorePath((Path) tmpDir + "/tmp", savedNAR); + addGit((Path) tmpDir + "/tmp"); + return; + } + } + + // FIXME: See if we can use the original source to reduce memory usage. + auto nar = make_ref(narSource.drain()); + /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ for (auto & ref : info.references) From a4f6f95512afd01b43392542fc2fc184e532b8ce Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 18:23:51 -0400 Subject: [PATCH 08/15] Add function for parsing git entry This lets us progressively fill in git dirs. --- src/libutil/git.cc | 68 +++++++++++++++++++++++++++------------------- src/libutil/git.hh | 10 +++++-- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 364d28e468a..7f76121fc8c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -22,18 +22,21 @@ using namespace std::string_literals; namespace nix { -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir); +static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir, + std::function addEntry); // Converts a Path to a ParseSink -void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir) { +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir, + std::function addEntry) { RestoreSink sink; sink.dstPath = path; parseGit(sink, source, realStoreDir, storeDir); } -void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir) +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir, + std::function addEntry) { - parse(sink, source, "", realStoreDir, storeDir); + parse(sink, source, "", realStoreDir, storeDir, addEntry); } static string getStringUntil(Source & source, char byte) @@ -65,7 +68,38 @@ static string getStoreEntry(const Path & storeDir, Hash hash, string name) return hash3.to_string(Base::Base32, false) + "-" + name; } -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir) +void addGitEntry(ParseSink & sink, const Path & path, + const Path & realStoreDir, const Path & storeDir, + int perm, std::string name, Hash hash) +{ + string entryName = getStoreEntry(storeDir, hash, "git"); + Path entry = absPath(realStoreDir + "/" + entryName); + + struct stat st; + if (lstat(entry.c_str(), &st)) + throw SysError("getting attributes of path '%1%'", entry); + + if (S_ISREG(st.st_mode)) { + if (perm == 40000) + throw SysError("file is a file but expected to be a directory '%1%'", entry); + + if (perm == 100755 || perm == 755) + sink.createExecutableFile(path + "/" + name); + else + sink.createRegularFile(path + "/" + name); + + sink.copyFile(entry); + } else if (S_ISDIR(st.st_mode)) { + if (perm != 40000) + throw SysError("file is a directory but expected to be a file '%1%'", entry); + + sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); + } else throw Error("file '%1%' has an unsupported type", entry); +} + +static void parse(ParseSink & sink, Source & source, const Path & path, + const Path & realStoreDir, const Path & storeDir, + std::function addEntry) { auto type = getString(source, 5); @@ -112,29 +146,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path, const Pa Hash hash(htSHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - string entryName = getStoreEntry(storeDir, hash, "git"); - Path entry = absPath(realStoreDir + "/" + entryName); - - struct stat st; - if (lstat(entry.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", entry); - - if (S_ISREG(st.st_mode)) { - if (perm == 40000) - throw SysError("file is a file but expected to be a directory '%1%'", entry); - - if (perm == 100755 || perm == 755) - sink.createExecutableFile(path + "/" + name); - else - sink.createRegularFile(path + "/" + name); - - sink.copyFile(entry); - } else if (S_ISDIR(st.st_mode)) { - if (perm != 40000) - throw SysError("file is a directory but expected to be a file '%1%'", entry); - - sink.copyDirectory(realStoreDir + "/" + entryName, path + "/" + name); - } else throw Error("file '%1%' has an unsupported type", entry); + addEntry(sink, path, realStoreDir, storeDir, perm, name, hash); } } else throw Error("input doesn't look like a Git object"); } diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 06a56d55b70..0df40c1f7e2 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -12,9 +12,15 @@ enum struct GitMode { Regular, }; -void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir); +void addGitEntry(ParseSink & sink, const Path & path, + const Path & realStoreDir, const Path & storeDir, + int perm, std::string name, Hash hash); -void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir); +void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir, + std::function addEntry = addGitEntry); + +void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir, + std::function addEntry = addGitEntry); // Dumps a single file to a sink GitMode dumpGitBlob(const Path & path, const struct stat st, Sink & sink); From 462ea0c411ae6e8af31c2cacfcf481b0a786e4be Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:06:03 -0400 Subject: [PATCH 09/15] Use SHA256 for addToStore This is for narHash, not the actual git object. --- src/libfetchers/git.cc | 2 +- src/libstore/local-store.cc | 8 +------- src/libstore/remote-store.cc | 3 --- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 49587cf461b..8e8238c62e7 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -403,7 +403,7 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, ingestionMethod, ingestionMethod == FileIngestionMethod::Git ? htSHA1 : htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, ingestionMethod, htSHA256, filter); // verify treeHash is what we actually obtained in the nix store if (input->treeHash) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a8ea6430d11..ff37a962dd9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1046,9 +1046,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) - throw Error("git ingestion must use sha1 hash"); - Hash h = hashString(hashAlgo, dump); // ugh... we need to calculate the hash just to get what path we @@ -1057,7 +1054,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam AutoDelete tmpDir(createTempDir(), true); StringSource source(dump); restorePath((Path) tmpDir + "/tmp", source); - h = dumpGitHash(hashAlgo, (Path) tmpDir + "/tmp"); + h = dumpGitHash(htSHA1, (Path) tmpDir + "/tmp"); } auto dstPath = makeFixedOutputPath(method, h, name); @@ -1121,9 +1118,6 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) - throw Error("git ingestion must use sha1 hash"); - if (method == FileIngestionMethod::Git) { // recursively add to store if path is a directory struct stat st; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4fccf47a122..cad171bc87d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -482,9 +482,6 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - if (method == FileIngestionMethod::Git && hashAlgo != htSHA1) - throw Error("git ingestion must use sha1 hash"); - Path srcPath(absPath(_srcPath)); // recursively add to store if path is a directory From 122a6333809243a972a438a2b16f07eb9107717d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:07:24 -0400 Subject: [PATCH 10/15] =?UTF-8?q?Don=E2=80=99t=20set=20nar=20values=20in?= =?UTF-8?q?=20ipfs=20binary=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/ipfs-binary-cache-store.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 6bf8a1e0835..7156ae05ccc 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -632,8 +632,6 @@ class IPFSBinaryCacheStore : public Store NarInfo narInfo { storePath }; narInfo.ca = ca; narInfo.url = "ipfs://" + *cid; - narInfo.narHash = std::get(*ca).hash; - narInfo.narSize = *size; (*callbackPtr)((std::shared_ptr) std::make_shared(narInfo)); return; From c63dba046765c77a0ce85127daf6dc431bbb4cf6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:13:51 -0400 Subject: [PATCH 11/15] fixup --- src/libutil/git.cc | 9 +++------ src/libutil/git.hh | 4 ++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 7f76121fc8c..0636d928a30 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -22,21 +22,18 @@ using namespace std::string_literals; namespace nix { -static void parse(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir, - std::function addEntry); - // Converts a Path to a ParseSink void restoreGit(const Path & path, Source & source, const Path & realStoreDir, const Path & storeDir, std::function addEntry) { RestoreSink sink; sink.dstPath = path; - parseGit(sink, source, realStoreDir, storeDir); + parseGit(sink, source, realStoreDir, storeDir, addEntry); } void parseGit(ParseSink & sink, Source & source, const Path & realStoreDir, const Path & storeDir, std::function addEntry) { - parse(sink, source, "", realStoreDir, storeDir, addEntry); + parseGitInternal(sink, source, "", realStoreDir, storeDir, addEntry); } static string getStringUntil(Source & source, char byte) @@ -97,7 +94,7 @@ void addGitEntry(ParseSink & sink, const Path & path, } else throw Error("file '%1%' has an unsupported type", entry); } -static void parse(ParseSink & sink, Source & source, const Path & path, +void parseGitInternal(ParseSink & sink, Source & source, const Path & path, const Path & realStoreDir, const Path & storeDir, std::function addEntry) { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 0df40c1f7e2..281290fe955 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -35,4 +35,8 @@ Hash dumpGitHash(HashType ht, const Path & path, PathFilter & filter = defaultPa void dumpGit(HashType ht, const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +void parseGitInternal(ParseSink & sink, Source & source, const Path & path, + const Path & realStoreDir, const Path & storeDir, + std::function addEntry); + } From d97c9e522ada3ea2db8388e0943185573ee7d0f7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:28:38 -0400 Subject: [PATCH 12/15] Handle converting git objects to nars --- src/libstore/ipfs-binary-cache-store.cc | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 7156ae05ccc..03f5630478a 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -495,6 +495,22 @@ class IPFSBinaryCacheStore : public Store } else throw Error("file '%1%' has an unsupported type", path); } + void getGitEntry(ParseSink & sink, const Path & path, + const Path & realStoreDir, const Path & storeDir, + int perm, std::string name, Hash hash) + { + auto url = "ipfs://f01781114" + hash.to_string(Base16, false); + auto source = sinkToSource([&](Sink & sink) { + getFile(url, sink); + }); + parseGitInternal(sink, *source, path + "/" + name, realStoreDir, storeDir, + [this] (ParseSink & sink, const Path & path, const Path & realStoreDir, const Path & storeDir, + int perm, std::string name, Hash hash) { + this->getGitEntry(sink, path, realStoreDir, storeDir, perm, name, hash); + }); + } + + public: void addToStore(const ValidPathInfo & info, Source & narSource, @@ -595,6 +611,21 @@ class IPFSBinaryCacheStore : public Store narSize += len; }); + // ugh... we have to convert git data to nar. + if (hasPrefix(info->url, "ipfs://f01781114")) { + AutoDelete tmpDir(createTempDir(), true); + auto source = sinkToSource([&](Sink & sink) { + getFile(info->url, sink); + }); + restoreGit((Path) tmpDir + "/tmp", *source, storeDir, storeDir, + [this] (ParseSink & sink, const Path & path, const Path & realStoreDir, const Path & storeDir, + int perm, std::string name, Hash hash) { + this->getGitEntry(sink, path, realStoreDir, storeDir, perm, name, hash); + }); + dumpPath((Path) tmpDir + "/tmp", wrapperSink); + return; + } + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); try { From 75ba0801bb5aec5f0faf7c753c574dfd5c1ea69c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:28:59 -0400 Subject: [PATCH 13/15] Add trustless ipfs mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this can be enabled via --substituters ipfs:// It doesn’t require any specific nar store - just takes it from the global ipfs store. --- src/libstore/ipfs-binary-cache-store.cc | 33 ++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 03f5630478a..7b87e6a2eea 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -44,6 +44,9 @@ class IPFSBinaryCacheStore : public Store }; Sync _state; + // only enable trustless operations + bool trustless = false; + public: IPFSBinaryCacheStore( @@ -60,12 +63,16 @@ class IPFSBinaryCacheStore : public Store sink << narVersionMagic1; narMagic = *sink.s; - if (cacheUri.back() == '/') + if (cacheUri.back() == '/' && cacheUri != "ipfs://") cacheUri.pop_back(); if (hasPrefix(cacheUri, "ipfs://")) { - initialIpfsPath = "/ipfs/" + std::string(cacheUri, 7); - state->ipfsPath = initialIpfsPath; + if (cacheUri == "ipfs://") + trustless = true; + else { + initialIpfsPath = "/ipfs/" + std::string(cacheUri, 7); + state->ipfsPath = initialIpfsPath; + } } else if (hasPrefix(cacheUri, "ipns://")) optIpnsPath = "/ipns/" + std::string(cacheUri, 7); else @@ -94,6 +101,9 @@ class IPFSBinaryCacheStore : public Store state->ipfsPath = initialIpfsPath; } + if (trustless) + return; + auto json = getIpfsDag(state->ipfsPath); // Verify StoreDir is correct @@ -218,6 +228,9 @@ class IPFSBinaryCacheStore : public Store { auto state(_state.lock()); + if (trustless) + return; + if (!optIpnsPath) { throw Error("We don't have an ipns path and the current ipfs address doesn't match the initial one.\n current: %s\n initial: %s", state->ipfsPath, initialIpfsPath); @@ -532,6 +545,9 @@ class IPFSBinaryCacheStore : public Store } } + if (trustless) + throw Error("cannot add '%s' to store because of trustless mode", printStorePath(info.path)); + // FIXME: See if we can use the original source to reduce memory usage. auto nar = make_ref(narSource.drain()); @@ -594,6 +610,9 @@ class IPFSBinaryCacheStore : public Store return true; } + if (trustless) + return false; + auto json = getIpfsDag(getIpfsPath()); if (!json.contains("nar")) return false; @@ -670,6 +689,11 @@ class IPFSBinaryCacheStore : public Store } } + if (trustless) { + (*callbackPtr)(nullptr); + return; + } + auto json = getIpfsDag(getIpfsPath()); if (!json.contains("nar") || !json["nar"].contains(storePath.to_string())) @@ -768,6 +792,9 @@ class IPFSBinaryCacheStore : public Store void addSignatures(const StorePath & storePath, const StringSet & sigs) override { + if (trustless) + throw Error("cannot add signatures available for '%s' because of trustless mode", printStorePath(storePath)); + /* Note: this is inherently racy since there is no locking on binary caches. In particular, with S3 this unreliable, even when addSignatures() is called sequentially on a path, because From 2daebd8285b991844539a3ef612e84d278b913b1 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:53:10 -0400 Subject: [PATCH 14/15] Separate ipfs objects from blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using block for getting nars doesn’t seem to work. unclear why, but has something to do with messing up the xz compression. --- src/libstore/ipfs-binary-cache-store.cc | 54 ++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 7b87e6a2eea..4ed7e3e75ce 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -374,6 +374,56 @@ class IPFSBinaryCacheStore : public Store void getIpfsObject(const std::string & ipfsPath, Callback> callback) noexcept + { + auto uri = daemonUri + "/api/v0/cat?arg=" + getFileTransfer()->urlEncode(ipfsPath); + + FileTransferRequest request(uri); + request.post = true; + request.tries = 1; + + auto callbackPtr = std::make_shared(std::move(callback)); + + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr](std::future result){ + try { + (*callbackPtr)(result.get().data); + } catch (FileTransferError & e) { + return (*callbackPtr)(std::shared_ptr()); + } catch (...) { + callbackPtr->rethrow(); + } + }} + ); + } + + void getIpfsBlock(const std::string & path, Sink & sink) + { + std::promise> promise; + getIpfsBlock(path, + {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink((unsigned char *) data->data(), data->size()); + } + + std::shared_ptr getIpfsBlock(const std::string & path) + { + StringSink sink; + try { + getIpfsBlock(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + } + + void getIpfsBlock(const std::string & ipfsPath, + Callback> callback) noexcept { auto uri = daemonUri + "/api/v0/block/get?arg=" + getFileTransfer()->urlEncode(ipfsPath); @@ -514,7 +564,7 @@ class IPFSBinaryCacheStore : public Store { auto url = "ipfs://f01781114" + hash.to_string(Base16, false); auto source = sinkToSource([&](Sink & sink) { - getFile(url, sink); + getIpfsBlock("/ipfs/" + std::string(url, 7), sink); }); parseGitInternal(sink, *source, path + "/" + name, realStoreDir, storeDir, [this] (ParseSink & sink, const Path & path, const Path & realStoreDir, const Path & storeDir, @@ -634,7 +684,7 @@ class IPFSBinaryCacheStore : public Store if (hasPrefix(info->url, "ipfs://f01781114")) { AutoDelete tmpDir(createTempDir(), true); auto source = sinkToSource([&](Sink & sink) { - getFile(info->url, sink); + getIpfsBlock("/ipfs/" + std::string(info->url, 7), sink); }); restoreGit((Path) tmpDir + "/tmp", *source, storeDir, storeDir, [this] (ParseSink & sink, const Path & path, const Path & realStoreDir, const Path & storeDir, From bd871a5c656acb8b7c70ea15c75f0591fdc981f0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 23 Jun 2020 19:57:13 -0400 Subject: [PATCH 15/15] Add test of adding things to the IPFS store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This combines the git ingestino with IPFS. We substitute using the ipfs:// so that we don’t accidentally pick up any nars. --- tests/ipfs.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index cc0cee4afba..25920f6bac0 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -140,3 +140,51 @@ nix copy $outPath --to ipns://$IPNS_ID --experimental-features nix-command # and copy back nix copy $outPath --store file://$IPFS_DST_IPNS_STORE --from ipns://$IPNS_ID --experimental-features nix-command + +# Verify git objects can be substituted correctly + +if [[ -n $(type -p git) ]]; then + repo=$TEST_ROOT/git + + rm -rf $repo $TEST_HOME/.cache/nix + + git init $repo + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" + + echo hello > $repo/hello + git -C $repo add hello + git -C $repo commit -m 'Bla1' + + treeHash=$(git -C $repo rev-parse HEAD:) + + path=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file://$repo; treeHash = \"$treeHash\"; }).outPath") + + # copy to ipfs in trustless mode, doesn’t require syncing + nix copy $path --to ipfs:// --experimental-features nix-command + + helloBlob=$(git -C $repo rev-parse HEAD:hello) + + ipfs block stat f01781114$treeHash + ipfs block get f01781114$treeHash > tree1 + (printf "tree %s\0" $(git -C $repo cat-file tree HEAD: | wc -c); git -C $repo cat-file tree HEAD:) > tree2 + diff tree1 tree2 + + ipfs block stat f01781114$helloBlob + ipfs block get f01781114$helloBlob > blob1 + (printf "blob %s\0" $(git -C $repo cat-file blob HEAD:hello | wc -c); git -C $repo cat-file blob HEAD:hello) > blob2 + diff blob1 blob2 + + clearStore + + # verify we can substitute from global ipfs store + path2=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$helloBlob\"; }).outPath" --substituters ipfs:// --option substitute true) + [[ "$(cat $path2)" = hello ]] + + path3=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = file:///no-such-repo; treeHash = \"$treeHash\"; }).outPath" --substituters ipfs:// --option substitute true) + + [[ "$(ls $path3)" = hello ]] + diff $path2 $path3/hello +else + echo "Git not installed; skipping IPFS/Git tests" +fi