diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c52..b2e37c5b522 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -721,9 +721,10 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); + TeeSource tee(from); + ParseSink sink; + parseDump(sink, tee); + saved = std::move(*tee.data); source = std::make_unique(saved); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb9da027dd6..0e33fb68784 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -77,8 +77,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + TeeSource tee(source); + ParseSink sink; + parseDump(sink, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -94,15 +95,15 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *tee.data); + info.narSize = tee.data->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; + auto source = StringSource { *tee.data }; addToStore(info, source, NoRepair, checkSigs, accessor); res.push_back(info.path); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b1fda633126..e93887f5c77 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -58,7 +58,6 @@ struct curlFileTransfer : public FileTransfer Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; unsigned int attempt = 0; @@ -125,7 +124,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -155,8 +154,18 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + decompressionSink = std::make_shared>>( + ref{ decompressionSink } + ); + } + } (*decompressionSink)((unsigned char *) contents, realSize); @@ -177,6 +186,7 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + std::string status; if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; auto ss = tokenizeString>(line, " "); @@ -420,14 +430,18 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr response; + if (decompressionSink) + if (auto teeSink = std::dynamic_pointer_cast>>(decompressionSink)) + response = teeSink->data; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, "%s of '%s' was interrupted", request.verb(), request.uri) + ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 - ? FileTransferError(err, "unable to %s '%s': HTTP error %d%s", + ? FileTransferError(err, response, "unable to %s '%s': HTTP error %d%s", request.verb(), request.uri, httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) - : FileTransferError(err, "unable to %s '%s': %s (%d)", + : FileTransferError(err, response, "unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code); /* If this is a transient error, then maybe retry the @@ -684,7 +698,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else @@ -849,6 +863,18 @@ std::string FileTransfer::urlEncode(const std::string & param) { throw URLEncodeError("not implemented"); } +template +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + if (response) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 47007b1d7e6..8a884d6bbd5 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -106,10 +106,12 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr response; // intentionally optional + template - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe25368d..32d98a6104d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,13 +63,6 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeSink : ParseSink -{ - TeeSource source; - - TeeSink(Source & source) : source(source) { } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index dd666a4e19f..1bd118b47ea 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -25,4 +25,16 @@ MakeError(UnknownCompressionMethod, Error); MakeError(CompressionError, Error); +template<> +struct TeeSink> : CompressionSink +{ + MAKE_TEE_SINK(ref); + void finish() override { + orig->finish(); + } + void write(const unsigned char * data, size_t len) override { + return orig->write(data, len); + } +}; + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1e6102ce1db..ac9d2e494b9 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -173,9 +173,8 @@ public: template SysError(const Args & ... args) - :Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512fc..88a6b7ffeb5 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -181,6 +181,28 @@ struct TeeSource : Source } }; +#define MAKE_TEE_SINK(T) \ + T orig; \ + ref data; \ + TeeSink(T && orig) \ + : orig(std::move(orig)), data(make_ref()) { } \ + void operator () (const unsigned char * data, size_t len) { \ + this->data->append((const char *) data, len); \ + (*this->orig)(data, len); \ + } \ + void operator () (const std::string & s) \ + { \ + *data += s; \ + (*this->orig)(s); \ + } + +template +struct TeeSink : Sink +{ + MAKE_TEE_SINK(T); +}; + + /* A reader that consumes the original Source until 'size'. */ struct SizedSource : Source {