From f1d60662a25b9c40b414748b2a66414ec36b9bfe Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 15 Jun 2020 15:42:40 -0400 Subject: [PATCH 01/16] Move ipfs-specific method to own functions --- src/libstore/ipfs-binary-cache-store.cc | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 43a5a664de1..a84e1c8cac2 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -191,7 +191,7 @@ class IPFSBinaryCacheStore : public BinaryCacheStore state->ipfsPath = "/ipfs/" + (std::string) json["Hash"]; } - void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) override + std::string addFile(const std::string & data) { // TODO: use callbacks @@ -199,10 +199,15 @@ class IPFSBinaryCacheStore : public BinaryCacheStore req.data = std::make_shared(data); req.post = true; req.tries = 1; + auto res = getFileTransfer()->upload(req); + auto json = nlohmann::json::parse(*res.data); + return (std::string) json["Hash"]; + } + + void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) override + { try { - auto res = getFileTransfer()->upload(req); - auto json = nlohmann::json::parse(*res.data); - addLink(path, "/ipfs/" + (std::string) json["Hash"]); + addLink(path, "/ipfs/" + addFile(data)); } catch (FileTransferError & e) { throw UploadToIPFS("while uploading to IPFS binary cache at '%s': %s", cacheUri, e.msg()); } @@ -211,7 +216,13 @@ class IPFSBinaryCacheStore : public BinaryCacheStore void getFile(const std::string & path, Callback> callback) noexcept override { - auto uri = daemonUri + "/api/v0/cat?arg=" + getFileTransfer()->urlEncode(getIpfsPath() + "/" + path); + getIpfsObject(getIpfsPath() + "/" + path, std::move(callback)); + } + + void getIpfsObject(const std::string & ipfsPath, + Callback> callback) noexcept + { + auto uri = daemonUri + "/api/v0/cat?arg=" + getFileTransfer()->urlEncode(ipfsPath); FileTransferRequest request(uri); request.post = true; From 4ca97246f89ecdc04aec725d61b04f43276dd8e3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 15 Jun 2020 15:44:48 -0400 Subject: [PATCH 02/16] Correctly specify visibility of ipfs-binary-cache-store --- src/libstore/ipfs-binary-cache-store.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index a84e1c8cac2..be458cdb2ca 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -84,7 +84,7 @@ class IPFSBinaryCacheStore : public BinaryCacheStore BinaryCacheStore::init(); } -protected: +private: // Given a ipns path, checks if it corresponds to a DNSLink path, and in // case returns the domain @@ -98,6 +98,8 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return std::nullopt; } +public: + bool fileExists(const std::string & path) override { auto uri = daemonUri + "/api/v0/object/stat?arg=" + getFileTransfer()->urlEncode(getIpfsPath() + "/" + path); @@ -173,6 +175,8 @@ class IPFSBinaryCacheStore : public BinaryCacheStore getFileTransfer()->download(req); } +private: + void addLink(std::string name, std::string ipfsObject) { auto state(_state.lock()); @@ -204,6 +208,8 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return (std::string) json["Hash"]; } +public: + void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) override { try { @@ -219,6 +225,8 @@ class IPFSBinaryCacheStore : public BinaryCacheStore getIpfsObject(getIpfsPath() + "/" + path, std::move(callback)); } +private: + void getIpfsObject(const std::string & ipfsPath, Callback> callback) noexcept { From 784e786527fad09d75cb7ae7f83c4566a21ae69b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 15 Jun 2020 15:46:55 -0400 Subject: [PATCH 03/16] Add ipfsObjectExists --- src/libstore/ipfs-binary-cache-store.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index be458cdb2ca..82e2f7d782c 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -98,11 +98,9 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return std::nullopt; } -public: - - bool fileExists(const std::string & path) override + bool ipfsObjectExists(const std::string ipfsPath) { - auto uri = daemonUri + "/api/v0/object/stat?arg=" + getFileTransfer()->urlEncode(getIpfsPath() + "/" + path); + auto uri = daemonUri + "/api/v0/object/stat?arg=" + getFileTransfer()->urlEncode(ipfsPath); FileTransferRequest request(uri); request.post = true; @@ -119,6 +117,13 @@ class IPFSBinaryCacheStore : public BinaryCacheStore } } +public: + + bool fileExists(const std::string & path) override + { + return ipfsObjectExists(getIpfsRootDir() + "/" + path); + } + // Resolve the IPNS name to an IPFS object std::string resolveIPNSName(std::string ipnsPath, bool offline) { debug("Resolving IPFS object of '%s', this could take a while.", ipnsPath); From ca1a565e93cd1073b597130a4846542d791c9906 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 15 Jun 2020 17:13:16 -0400 Subject: [PATCH 04/16] Fixup ipfs-binary-cache-store modifiers --- src/libstore/ipfs-binary-cache-store.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 82e2f7d782c..a4ceee92037 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -88,7 +88,8 @@ class IPFSBinaryCacheStore : public BinaryCacheStore // Given a ipns path, checks if it corresponds to a DNSLink path, and in // case returns the domain - std::optional isDNSLinkPath(std::string path) { + static std::optional isDNSLinkPath(std::string path) + { if (path.find("/ipns/") != 0) throw Error("The provided path is not a ipns path"); auto subpath = std::string(path, 6); @@ -117,15 +118,17 @@ class IPFSBinaryCacheStore : public BinaryCacheStore } } -public: +protected: bool fileExists(const std::string & path) override { return ipfsObjectExists(getIpfsRootDir() + "/" + path); } +private: + // Resolve the IPNS name to an IPFS object - std::string resolveIPNSName(std::string ipnsPath, bool offline) { + static std::string resolveIPNSName(std::string ipnsPath, bool offline) { debug("Resolving IPFS object of '%s', this could take a while.", ipnsPath); auto uri = daemonUri + "/api/v0/name/resolve?offline=" + (offline?"true":"false") + "&arg=" + getFileTransfer()->urlEncode(ipnsPath); FileTransferRequest request(uri); @@ -138,6 +141,8 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return json["Path"]; } +public: + // IPNS publish can be slow, we try to do it rarely. void sync() override { @@ -213,7 +218,7 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return (std::string) json["Hash"]; } -public: +protected: void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) override { From 054cabfab90b7102a75b0ebe194c32fae4670219 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 15 Jun 2020 17:35:13 -0400 Subject: [PATCH 05/16] Add dag helper methods --- src/libstore/ipfs-binary-cache-store.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index a4ceee92037..7b2db46133e 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -86,6 +86,26 @@ class IPFSBinaryCacheStore : public BinaryCacheStore private: + std::string putIpfsDag(std::string data) + { + auto req = FileTransferRequest(daemonUri + "/api/v0/dag/put"); + req.data = std::make_shared(data); + req.post = true; + req.tries = 1; + auto res = getFileTransfer()->upload(req); + auto json = nlohmann::json::parse(*res.data); + return json["Cid"]["/"]; + } + + std::string getIpfsDag(std::string objectPath) + { + auto req = FileTransferRequest(daemonUri + "/api/v0/dag/get?arg=" + objectPath); + req.post = true; + req.tries = 1; + auto res = getFileTransfer()->download(req); + return *res.data; + } + // Given a ipns path, checks if it corresponds to a DNSLink path, and in // case returns the domain static std::optional isDNSLinkPath(std::string path) @@ -122,13 +142,13 @@ class IPFSBinaryCacheStore : public BinaryCacheStore bool fileExists(const std::string & path) override { - return ipfsObjectExists(getIpfsRootDir() + "/" + path); + return ipfsObjectExists(getIpfsPath() + "/" + path); } private: // Resolve the IPNS name to an IPFS object - static std::string resolveIPNSName(std::string ipnsPath, bool offline) { + std::string resolveIPNSName(std::string ipnsPath, bool offline) { debug("Resolving IPFS object of '%s', this could take a while.", ipnsPath); auto uri = daemonUri + "/api/v0/name/resolve?offline=" + (offline?"true":"false") + "&arg=" + getFileTransfer()->urlEncode(ipnsPath); FileTransferRequest request(uri); From df3aea2e1f37189526950e77413c1b0151df0262 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 16 Jun 2020 13:26:04 -0400 Subject: [PATCH 06/16] Implement Store directly instead of via BinaryCacheStore This leads to some duplication, but we will slowly remove that as we continue to add IPLD meta info. --- src/libstore/ipfs-binary-cache-store.cc | 309 ++++++++++++++++++++++-- 1 file changed, 292 insertions(+), 17 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 7b2db46133e..0e1d95c278e 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -4,16 +4,28 @@ #include "binary-cache-store.hh" #include "filetransfer.hh" #include "nar-info-disk-cache.hh" +#include "archive.hh" +#include "compression.hh" namespace nix { MakeError(UploadToIPFS, Error); -class IPFSBinaryCacheStore : public BinaryCacheStore +class IPFSBinaryCacheStore : public Store { +public: + + const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting parallelCompression{this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; + private: + std::unique_ptr secretKey; + std::string narMagic; + std::string cacheUri; std::string daemonUri; @@ -34,11 +46,18 @@ class IPFSBinaryCacheStore : public BinaryCacheStore IPFSBinaryCacheStore( const Params & params, const Path & _cacheUri) - : BinaryCacheStore(params) + : Store(params) , cacheUri(_cacheUri) { auto state(_state.lock()); + if (secretKeyFile != "") + secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); + + StringSink sink; + sink << narVersionMagic1; + narMagic = *sink.s; + if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -76,14 +95,6 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return cacheUri; } - void init() override - { - std::string cacheInfoFile = "nix-cache-info"; - if (!fileExists(cacheInfoFile)) - upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); - BinaryCacheStore::init(); - } - private: std::string putIpfsDag(std::string data) @@ -140,7 +151,7 @@ class IPFSBinaryCacheStore : public BinaryCacheStore protected: - bool fileExists(const std::string & path) override + bool fileExists(const std::string & path) { return ipfsObjectExists(getIpfsPath() + "/" + path); } @@ -238,9 +249,7 @@ class IPFSBinaryCacheStore : public BinaryCacheStore return (std::string) json["Hash"]; } -protected: - - void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) override + void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) { try { addLink(path, "/ipfs/" + addFile(data)); @@ -250,12 +259,36 @@ class IPFSBinaryCacheStore : public BinaryCacheStore } void getFile(const std::string & path, - Callback> callback) noexcept override + Callback> callback) noexcept { getIpfsObject(getIpfsPath() + "/" + path, std::move(callback)); } -private: + void getFile(const std::string & path, Sink & sink) + { + std::promise> promise; + getFile(path, + {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink((unsigned char *) data->data(), data->size()); + } + + std::shared_ptr getFile(const std::string & path) + { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + } void getIpfsObject(const std::string & ipfsPath, Callback> callback) noexcept @@ -281,6 +314,249 @@ class IPFSBinaryCacheStore : public BinaryCacheStore ); } + std::string narInfoFileFor(const StorePath & storePath) + { + return storePathToHash(printStorePath(storePath)) + ".narinfo"; + } + + void writeNarInfo(ref narInfo) + { + auto narInfoFile = narInfoFileFor(narInfo->path); + + upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); + + auto hashPart = storePathToHash(printStorePath(narInfo->path)); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr(narInfo) }); + } + } + +public: + + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override + { + // FIXME: See if we can use the original source to reduce memory usage. + auto nar = make_ref(narSource.drain()); + + if (!repair && isValidPath(info.path)) return; + + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) + try { + if (ref != info.path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", + printStorePath(info.path), printStorePath(ref)); + } + + assert(nar->compare(0, narMagic.size(), narMagic) == 0); + + auto narInfo = make_ref(info); + + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); + + if (info.narHash && info.narHash != narInfo->narHash) + throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); + + /* Compress the NAR. */ + narInfo->compression = compression; + auto now1 = std::chrono::steady_clock::now(); + auto narCompressed = compress(compression, *nar, parallelCompression); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); + + auto duration = std::chrono::duration_cast(now2 - now1).count(); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), narInfo->narSize, + ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), + duration); + + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); + + /* Atomically write the NAR file. */ + if (repair || !fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + } else + stats.narWriteAverted++; + + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) narInfo->sign(*this, *secretKey); + + writeNarInfo(narInfo); + + stats.narInfoWrite++; + } + + bool isValidPathUncached(const StorePath & storePath) override + { + return fileExists(narInfoFileFor(storePath)); + } + + void narFromPath(const StorePath & storePath, Sink & sink) override + { + auto info = queryPathInfo(storePath).cast(); + + uint64_t narSize = 0; + + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + narSize += len; + }); + + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + + try { + getFile(info->url, *decompressor); + } catch (NoSuchBinaryCacheFile & e) { + throw SubstituteGone(e.what()); + } + + decompressor->finish(); + + stats.narRead++; + //stats.narReadCompressedBytes += nar->size(); // FIXME + stats.narReadBytes += narSize; + } + + void queryPathInfoUncached(const StorePath & storePath, + Callback> callback) noexcept override + { + auto uri = getUri(); + auto storePathS = printStorePath(storePath); + auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, + fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); + PushActivity pact(act->id); + + auto narInfoFile = narInfoFileFor(storePath); + + auto callbackPtr = std::make_shared(std::move(callback)); + + getFile(narInfoFile, + {[=](std::future> fut) { + try { + auto data = fut.get(); + + if (!data) return (*callbackPtr)(nullptr); + + stats.narInfoRead++; + + (*callbackPtr)((std::shared_ptr) + std::make_shared(*this, *data, narInfoFile)); + + (void) act; // force Activity into this lambda to ensure it stays alive + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override + { + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (method == FileIngestionMethod::Recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info(makeFixedOutputPath(method, h, name)); + + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); + + return std::move(info.path); + } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override + { + ValidPathInfo info(computeStorePathForText(name, s, references)); + info.references = cloneStorePathSet(references); + + if (repair || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); + } + + return std::move(info.path); + } + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override + { + /* Note: this is inherently racy since there is no locking on + binary caches. In particular, with S3 this unreliable, even + when addSignatures() is called sequentially on a path, because + S3 might return an outdated cached version. */ + + auto narInfo = make_ref((NarInfo &) *queryPathInfo(storePath)); + + narInfo->sigs.insert(sigs.begin(), sigs.end()); + + auto narInfoFile = narInfoFileFor(narInfo->path); + + writeNarInfo(narInfo); + } + + std::shared_ptr getBuildLog(const StorePath & path) override + { + auto drvPath = path.clone(); + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + // FIXME: add a "Log" field to .narinfo + if (!info->deriver) return nullptr; + drvPath = info->deriver->clone(); + } catch (InvalidPath &) { + return nullptr; + } + } + + auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + + debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); + + return getFile(logPath); + } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { unsupported("buildDerivation"); } + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + }; static RegisterStoreImplementation regStore([]( @@ -291,7 +567,6 @@ static RegisterStoreImplementation regStore([]( uri.substr(0, strlen("ipns://")) != "ipns://") return 0; auto store = std::make_shared(params, uri); - store->init(); return store; }); From 213ded33b52a5adbbe9a177dd904c62f8eed500c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 16 Jun 2020 14:01:34 -0400 Subject: [PATCH 07/16] Remove offline from publish --- src/libstore/ipfs-binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 0e1d95c278e..bdc8e787531 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -207,7 +207,7 @@ class IPFSBinaryCacheStore : public Store debug("Publishing '%s' to '%s', this could take a while.", state->ipfsPath, ipnsPath); - auto uri = daemonUri + "/api/v0/name/publish?offline=true&arg=" + getFileTransfer()->urlEncode(state->ipfsPath); + auto uri = daemonUri + "/api/v0/name/publish?arg=" + getFileTransfer()->urlEncode(state->ipfsPath); uri += "&key=" + std::string(ipnsPath, 6); auto req = FileTransferRequest(uri); From 93d71aed2042e01692e69c1c81bd45e7c7062f58 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 16 Jun 2020 18:35:17 -0400 Subject: [PATCH 08/16] Use IPLD instead of UnixFS This lets us have some more structure to our data. We can store key value for meta and narinfos. --- src/libstore/ipfs-binary-cache-store.cc | 210 ++++++++++++++++-------- 1 file changed, 142 insertions(+), 68 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index bdc8e787531..00b04a9ce5c 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -88,6 +88,22 @@ class IPFSBinaryCacheStore : public Store initialIpfsPath = resolveIPNSName(ipnsPath, true); state->ipfsPath = initialIpfsPath; } + + auto json = getIpfsDag(state->ipfsPath); + + // Verify StoreDir is correct + if (json.find("StoreDir") == json.end()) { + json["StoreDir"] = storeDir; + state->ipfsPath = putIpfsDag(json); + } else if (json["StoreDir"] != storeDir) + throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") + % getUri() % json["StoreDir"] % storeDir); + + if (json.find("WantMassQuery") != json.end()) + wantMassQuery.setDefault(json["WantMassQuery"] ? "true" : "false"); + + if (json.find("Priority") != json.end()) + priority.setDefault(fmt("%d", json["Priority"])); } std::string getUri() override @@ -97,24 +113,27 @@ class IPFSBinaryCacheStore : public Store private: - std::string putIpfsDag(std::string data) + std::string putIpfsDag(nlohmann::json data) { auto req = FileTransferRequest(daemonUri + "/api/v0/dag/put"); - req.data = std::make_shared(data); + req.data = std::make_shared(data.dump()); req.post = true; req.tries = 1; auto res = getFileTransfer()->upload(req); auto json = nlohmann::json::parse(*res.data); - return json["Cid"]["/"]; + return "/ipfs/" + (std::string) json["Cid"]["/"]; } - std::string getIpfsDag(std::string objectPath) + nlohmann::json getIpfsDag(std::string objectPath) { + debug("get ipfs dag %s", objectPath); + auto req = FileTransferRequest(daemonUri + "/api/v0/dag/get?arg=" + objectPath); req.post = true; req.tries = 1; auto res = getFileTransfer()->download(req); - return *res.data; + auto json = nlohmann::json::parse(*res.data); + return json; } // Given a ipns path, checks if it corresponds to a DNSLink path, and in @@ -149,15 +168,11 @@ class IPFSBinaryCacheStore : public Store } } -protected: - bool fileExists(const std::string & path) { return ipfsObjectExists(getIpfsPath() + "/" + path); } -private: - // Resolve the IPNS name to an IPFS object std::string resolveIPNSName(std::string ipnsPath, bool offline) { debug("Resolving IPFS object of '%s', this could take a while.", ipnsPath); @@ -261,7 +276,10 @@ class IPFSBinaryCacheStore : public Store void getFile(const std::string & path, Callback> callback) noexcept { - getIpfsObject(getIpfsPath() + "/" + path, std::move(callback)); + std::string path_(path); + if (hasPrefix(path, "ipfs://")) + path_ = "/ipfs/" + std::string(path, 7); + getIpfsObject(path_, std::move(callback)); } void getFile(const std::string & path, Sink & sink) @@ -314,21 +332,68 @@ class IPFSBinaryCacheStore : public Store ); } - std::string narInfoFileFor(const StorePath & storePath) - { - return storePathToHash(printStorePath(storePath)) + ".narinfo"; - } - void writeNarInfo(ref narInfo) { - auto narInfoFile = narInfoFileFor(narInfo->path); + auto json = nlohmann::json::object(); + json["narHash"] = narInfo->narHash.to_string(Base32); + json["narSize"] = narInfo->narSize; + + auto narMap = getIpfsDag(getIpfsPath())["nar"]; + + json["references"] = nlohmann::json::array(); + for (auto & ref : narInfo->references) { + if (ref == narInfo->path) { + json["references"].push_back(printStorePath(ref)); + } else { + json["references"].push_back(narMap[printStorePath(ref)]); + } + } - upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); + if (narInfo->ca != "") + json["ca"] = narInfo->ca; - auto hashPart = storePathToHash(printStorePath(narInfo->path)); + if (narInfo->deriver) + json["deriver"] = printStorePath(*narInfo->deriver); + + if (narInfo->registrationTime) + json["registrationTime"] = narInfo->registrationTime; + + if (narInfo->ultimate) + json["ultimate"] = narInfo->ultimate; + + if (!narInfo->sigs.empty()) { + json["sigs"] = nlohmann::json::array(); + for (auto & sig : narInfo->sigs) + json["sigs"].push_back(sig); + } + + if (!narInfo->url.empty()) { + json["url"] = nlohmann::json::object(); + json["url"]["/"] = std::string(narInfo->url, 7); + } + if (narInfo->fileHash) + json["downloadHash"] = narInfo->fileHash.to_string(); + if (narInfo->fileSize) + json["downloadSize"] = narInfo->fileSize; + + auto narObjectPath = putIpfsDag(json); + + auto state(_state.lock()); + json = getIpfsDag(state->ipfsPath); + + if (json.find("nar") == json.end()) + json["nar"] = nlohmann::json::object(); + + auto hashObject = nlohmann::json::object(); + hashObject.emplace("/", std::string(narObjectPath, 6)); + + json["nar"].emplace(printStorePath(narInfo->path), hashObject); + + state->ipfsPath = putIpfsDag(json); { - auto state_(state.lock()); + auto hashPart = storePathToHash(printStorePath(narInfo->path)); + auto state_(this->state.lock()); state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr(narInfo) }); } } @@ -378,18 +443,9 @@ class IPFSBinaryCacheStore : public Store ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - /* Atomically write the NAR file. */ - if (repair || !fileExists(narInfo->url)) { - stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); - } else - stats.narWriteAverted++; + stats.narWrite++; + narInfo->url = "ipfs://" + addFile(*narCompressed); stats.narWriteBytes += nar->size(); stats.narWriteCompressedBytes += narCompressed->size(); @@ -405,7 +461,10 @@ class IPFSBinaryCacheStore : public Store bool isValidPathUncached(const StorePath & storePath) override { - return fileExists(narInfoFileFor(storePath)); + auto json = getIpfsDag(getIpfsPath()); + if (!json.contains("nar")) + return false; + return json["nar"].contains(printStorePath(storePath)); } void narFromPath(const StorePath & storePath, Sink & sink) override @@ -437,33 +496,69 @@ class IPFSBinaryCacheStore : public Store void queryPathInfoUncached(const StorePath & storePath, Callback> callback) noexcept override { + // TODO: properly use callbacks + + auto callbackPtr = std::make_shared(std::move(callback)); + auto uri = getUri(); auto storePathS = printStorePath(storePath); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); - auto narInfoFile = narInfoFileFor(storePath); + auto json = getIpfsDag(getIpfsPath()); - auto callbackPtr = std::make_shared(std::move(callback)); + if (!json.contains("nar") || !json["nar"].contains(printStorePath(storePath))) + return (*callbackPtr)(nullptr); - getFile(narInfoFile, - {[=](std::future> fut) { - try { - auto data = fut.get(); + auto narObjectHash = (std::string) json["nar"][printStorePath(storePath)]["/"]; + json = getIpfsDag("/ipfs/" + narObjectHash); - if (!data) return (*callbackPtr)(nullptr); + NarInfo narInfo(storePath.clone()); + narInfo.narHash = Hash((std::string) json["narHash"]); + narInfo.narSize = json["narSize"]; - stats.narInfoRead++; + auto narMap = getIpfsDag(getIpfsPath())["nar"]; + for (auto & ref : json["references"]) { + if (ref.type() == nlohmann::json::value_t::object) { + for (auto & v : narMap.items()) { + if (v.value() == ref) { + narInfo.references.insert(parseStorePath(v.key())); + break; + } + } + } else if (ref.type() == nlohmann::json::value_t::string) + narInfo.references.insert(parseStorePath((std::string) ref)); + } - (*callbackPtr)((std::shared_ptr) - std::make_shared(*this, *data, narInfoFile)); + if (json.find("ca") != json.end()) + narInfo.ca = json["ca"]; - (void) act; // force Activity into this lambda to ensure it stays alive - } catch (...) { - callbackPtr->rethrow(); - } - }}); + if (json.find("deriver") != json.end()) + narInfo.deriver = parseStorePath((std::string) json["deriver"]); + + if (json.find("registrationTime") != json.end()) + narInfo.registrationTime = json["registrationTime"]; + + if (json.find("ultimate") != json.end()) + narInfo.ultimate = json["ultimate"]; + + if (json.find("sigs") != json.end()) + for (auto & sig : json["sigs"]) + narInfo.sigs.insert((std::string) sig); + + if (json.find("url") != json.end()) { + narInfo.url = "/ipfs/" + json["url"]["/"]; + } + + if (json.find("downloadHash") != json.end()) + narInfo.fileHash = Hash((std::string) json["downloadHash"]); + + if (json.find("downloadSize") != json.end()) + narInfo.fileSize = json["downloadSize"]; + + (*callbackPtr)((std::shared_ptr) + std::make_shared(narInfo)); } StorePath addToStore(const string & name, const Path & srcPath, @@ -520,32 +615,11 @@ class IPFSBinaryCacheStore : public Store narInfo->sigs.insert(sigs.begin(), sigs.end()); - auto narInfoFile = narInfoFileFor(narInfo->path); - writeNarInfo(narInfo); } std::shared_ptr getBuildLog(const StorePath & path) override - { - auto drvPath = path.clone(); - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return nullptr; - drvPath = info->deriver->clone(); - } catch (InvalidPath &) { - return nullptr; - } - } - - auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); - - debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); - - return getFile(logPath); - } + { unsupported("getBuildLog"); } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override From d1fc0770e3e6c1c738f36262d8709c37b66987a0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 16 Jun 2020 18:36:05 -0400 Subject: [PATCH 09/16] Make error message more idiomatic --- src/libstore/ipfs-binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 00b04a9ce5c..27366cc4e4d 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -141,7 +141,7 @@ class IPFSBinaryCacheStore : public Store static std::optional isDNSLinkPath(std::string path) { if (path.find("/ipns/") != 0) - throw Error("The provided path is not a ipns path"); + throw Error("path '%s' is not an ipns path", path); auto subpath = std::string(path, 6); if (subpath.find(".") != std::string::npos) { return subpath; From 7fa8391f3f6aafb6b1540b5e078e32bb5454e574 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 10:55:59 -0400 Subject: [PATCH 10/16] Properly store compression / system in ipfs binary cache These values were originally missing in the json. They are needed to properly decompress the NAR. --- src/libstore/ipfs-binary-cache-store.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index bdcda86861f..ef259966e6f 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -126,8 +126,6 @@ class IPFSBinaryCacheStore : public Store nlohmann::json getIpfsDag(std::string objectPath) { - debug("get ipfs dag %s", objectPath); - auto req = FileTransferRequest(daemonUri + "/api/v0/dag/get?arg=" + objectPath); req.post = true; req.tries = 1; @@ -376,6 +374,9 @@ class IPFSBinaryCacheStore : public Store if (narInfo->fileSize) json["downloadSize"] = narInfo->fileSize; + json["compression"] = narInfo->compression; + json["system"] = narInfo->system; + auto narObjectPath = putIpfsDag(json); auto state(_state.lock()); @@ -549,9 +550,8 @@ class IPFSBinaryCacheStore : public Store for (auto & sig : json["sigs"]) narInfo.sigs.insert((std::string) sig); - if (json.find("url") != json.end()) { - narInfo.url = "/ipfs/" + json["url"]["/"].get(); - } + if (json.find("url") != json.end()) + narInfo.url = "ipfs://" + json["url"]["/"].get(); if (json.find("downloadHash") != json.end()) narInfo.fileHash = Hash((std::string) json["downloadHash"]); @@ -559,6 +559,12 @@ class IPFSBinaryCacheStore : public Store if (json.find("downloadSize") != json.end()) narInfo.fileSize = json["downloadSize"]; + if (json.find("compression") != json.end()) + narInfo.compression = json["compression"]; + + if (json.find("system") != json.end()) + narInfo.system = json["system"]; + (*callbackPtr)((std::shared_ptr) std::make_shared(narInfo)); } From 5caf620381dd809b08f0c881cc708f108b1e2a75 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 16:53:53 -0400 Subject: [PATCH 11/16] Increment CURLOPT_EXPECT_100_TIMEOUT_MS to 5 min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Curl’s Expect: 100-continue seems to mess up IPFS long response. I’m unsure why, but it may have something to do with how Nix is handling threads. Here, I set it to 5 minutes which should never be reached --- src/libstore/filetransfer.cc | 2 ++ src/libstore/ipfs-binary-cache-store.cc | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b760c45671d..586ccbe14a1 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -324,6 +324,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_EXPECT_100_TIMEOUT_MS, 300000); + /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index ef259966e6f..b35322748ba 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -85,7 +85,7 @@ class IPFSBinaryCacheStore : public Store // Resolve the IPNS name to an IPFS object if (optIpnsPath) { auto ipnsPath = *optIpnsPath; - initialIpfsPath = resolveIPNSName(ipnsPath, true); + initialIpfsPath = resolveIPNSName(ipnsPath); state->ipfsPath = initialIpfsPath; } @@ -172,9 +172,9 @@ class IPFSBinaryCacheStore : public Store } // Resolve the IPNS name to an IPFS object - std::string resolveIPNSName(std::string ipnsPath, bool offline) { + std::string resolveIPNSName(std::string ipnsPath) { debug("Resolving IPFS object of '%s', this could take a while.", ipnsPath); - auto uri = daemonUri + "/api/v0/name/resolve?offline=" + (offline?"true":"false") + "&arg=" + getFileTransfer()->urlEncode(ipnsPath); + auto uri = daemonUri + "/api/v0/name/resolve?arg=" + getFileTransfer()->urlEncode(ipnsPath); FileTransferRequest request(uri); request.post = true; request.tries = 1; @@ -199,7 +199,7 @@ class IPFSBinaryCacheStore : public Store auto ipnsPath = *optIpnsPath; - auto resolvedIpfsPath = resolveIPNSName(ipnsPath, false); + auto resolvedIpfsPath = resolveIPNSName(ipnsPath); if (resolvedIpfsPath != initialIpfsPath) { throw Error("The IPNS hash or DNS link %s resolves now to something different from the value it had when Nix was started;\n wanted: %s\n got %s\nPerhaps something else updated it in the meantime?", initialIpfsPath, resolvedIpfsPath); From 2c89c62d1992d26396a0119c0c19c638845dcca2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 15:24:03 -0400 Subject: [PATCH 12/16] Set allow-offline=true on publish this is needed for our tests to succeed --- src/libstore/ipfs-binary-cache-store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index b35322748ba..924d5d45b15 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -220,7 +220,8 @@ class IPFSBinaryCacheStore : public Store debug("Publishing '%s' to '%s', this could take a while.", state->ipfsPath, ipnsPath); - auto uri = daemonUri + "/api/v0/name/publish?arg=" + getFileTransfer()->urlEncode(state->ipfsPath); + auto uri = daemonUri + "/api/v0/name/publish?allow-offline=true"; + uri += "&arg=" + getFileTransfer()->urlEncode(state->ipfsPath); uri += "&key=" + std::string(ipnsPath, 6); auto req = FileTransferRequest(uri); From ed0e05d8c939e6c4f45c5aff8306002075aebaf4 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 15:57:42 -0400 Subject: [PATCH 13/16] Add note regarding key lookup --- src/libstore/ipfs-binary-cache-store.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 03c5768612f..3481ff2be8e 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -227,6 +227,9 @@ class IPFSBinaryCacheStore : public Store // `ipfs key list` command, so that we publish to the right address in // case the user has multiple ones available. + // NOTE: this is needed for ipfs < 0.5.0 because key must be a + // name, not an address. + auto ipnsPathHash = std::string(ipnsPath, 6); debug("Getting the name corresponding to hash %s", ipnsPathHash); From 81abf59a181c53bf836cd78f309b89aae4dea3f1 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 15:57:51 -0400 Subject: [PATCH 14/16] Verify a derivation with dependencies works in ipfs --- tests/ipfs.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 8acff2c2dbb..6a554ad3561 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -113,3 +113,11 @@ DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ --no-out-link \ -j0 \ --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) + +# Verify we can copy something with dependencies +outPath=$(nix-build dependencies.nix --no-out-link) + +nix copy $outPath --to ipns://$IPNS_ID --experimental-features nix-command + +# and copy back +nix copy $outPath --store file://$IPFS_DST_IPNS_STORE --from ipns://$IPNS_ID --experimental-features nix-command From ba7a4b75a010717c1ea2c31e46f2792af2fe132e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 16:32:17 -0400 Subject: [PATCH 15/16] Readd ipfs gateway tests --- tests/ipfs.sh | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/ipfs.sh b/tests/ipfs.sh index 6a554ad3561..cc0cee4afba 100644 --- a/tests/ipfs.sh +++ b/tests/ipfs.sh @@ -38,6 +38,8 @@ mkdir $IPFS_TESTS # method) IPFS_SRC_STORE=$IPFS_TESTS/ipfs_source_store +IPFS_DST_HTTP_STORE=$IPFS_TESTS/ipfs_dest_http_store +IPFS_DST_HTTP_LOCAL_STORE=$IPFS_TESTS/ipfs_dest_http_local_store IPFS_DST_IPFS_STORE=$IPFS_TESTS/ipfs_dest_ipfs_store IPFS_DST_IPNS_STORE=$IPFS_TESTS/ipfs_dest_ipns_store @@ -70,6 +72,23 @@ for path in $storePaths; do done unset path +MANUAL_IPFS_HASH=$(ipfs add -r $IPFS_SRC_STORE 2>/dev/null | tail -n 1 | awk '{print $2}') + +################################################################################ +## Create the local http store and download the derivation there +################################################################################ + +mkdir $IPFS_DST_HTTP_LOCAL_STORE + +IPFS_HTTP_LOCAL_PREFIX='http://localhost:8080/ipfs' + +nix-build ./fixed.nix -A good \ + --option substituters $IPFS_HTTP_LOCAL_PREFIX/$MANUAL_IPFS_HASH \ + --store $IPFS_DST_HTTP_LOCAL_STORE \ + --no-out-link \ + -j0 \ + --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) + ################################################################################ ## Create the ipfs store and download the derivation there ################################################################################ @@ -87,12 +106,12 @@ nix copy --to ipfs://$IPFS_HASH $(nix-build ./fixed.nix -A good) --experimental- mkdir $IPFS_DST_IPFS_STORE -DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ +nix-build ./fixed.nix -A good \ --option substituters 'ipfs://'$IPFS_HASH \ --store $IPFS_DST_IPFS_STORE \ --no-out-link \ -j0 \ - --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) + --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) ################################################################################ @@ -107,12 +126,12 @@ nix copy --to ipns://$IPNS_ID $(nix-build ./fixed.nix -A good) --experimental-fe mkdir $IPFS_DST_IPNS_STORE -DOWNLOAD_LOCATION=$(nix-build ./fixed.nix -A good \ +nix-build ./fixed.nix -A good \ --option substituters 'ipns://'$IPNS_ID \ --store $IPFS_DST_IPNS_STORE \ --no-out-link \ -j0 \ - --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE)) + --option trusted-public-keys $(cat $SIGNING_KEY_PUB_FILE) # Verify we can copy something with dependencies outPath=$(nix-build dependencies.nix --no-out-link) From 9e4268d0cb440c0894ccd5303c171eaad82132bd Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 18 Jun 2020 16:34:23 -0400 Subject: [PATCH 16/16] Add FIXME for CURLOPT_EXPECT_100_TIMEOUT_MS --- src/libstore/filetransfer.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8579d14ac3b..1f485d18c88 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -325,6 +325,12 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); + /* FIXME: We hit a weird issue when 1 second goes by + * without Expect: 100-continue. curl_multi_perform + * appears to block indefinitely. To workaround this, we + * just set the timeout to a really big value unlikely to + * be hit in any server without Expect: 100-continue. This + * may specifically be a bug in the IPFS API. */ curl_easy_setopt(req, CURLOPT_EXPECT_100_TIMEOUT_MS, 300000); /* If no file exist in the specified path, curl continues to work