diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index d668b2c1a4..101e712c78 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -19,7 +19,6 @@ add_library ( src/context.cpp src/credentials/credentials.cpp src/credentials/policy/policies.cpp - src/http/http.cpp src/http/policy.cpp src/http/request.cpp src/http/response.cpp diff --git a/sdk/core/azure-core/inc/http/curl/curl.hpp b/sdk/core/azure-core/inc/http/curl/curl.hpp index d50a078dec..6e3af7e4f6 100644 --- a/sdk/core/azure-core/inc/http/curl/curl.hpp +++ b/sdk/core/azure-core/inc/http/curl/curl.hpp @@ -14,7 +14,7 @@ namespace Azure { namespace Core { namespace Http { - constexpr auto UploadSstreamPageSize = 1024 * 64; + constexpr auto UploadStreamPageSize = 1024 * 64; constexpr auto LibcurlReaderSize = 100; /** @@ -43,6 +43,18 @@ namespace Azure { namespace Core { namespace Http { EndOfHeaders, }; + /** + * @brief Defines the strategy to read the body from an HTTP Response + * + */ + enum class ResponseBodyLengthType + { + ContentLength, + Chunked, + ReadToCloseConnection, + NoBody, + }; + /** * @brief stateful component used to read and parse a buffer to construct a valid HTTP Response. * @@ -212,6 +224,20 @@ namespace Azure { namespace Core { namespace Http { */ size_t m_bodyStartInBuffer; + /** + * @brief Control field to handle the number of bytes containing relevant data within the + * internal buffer. This is because internal buffer can be set to be size N but after writing + * from wire into it, it can be holding less then N bytes. + * + */ + size_t m_innerBufferSize; + + /** + * @brief Defines the strategy to read a body from an HTTP Response + * + */ + ResponseBodyLengthType m_bodyLengthType; + /** * @brief This is a copy of the value of an HTTP response header `content-length`. The value is * received as string and parsed to size_t. This field avoid parsing the string header everytime @@ -329,6 +355,7 @@ namespace Azure { namespace Core { namespace Http { * requested. */ uint64_t ReadSocketToBuffer(uint8_t* buffer, size_t bufferSize); + uint64_t ReadChunkedBody(uint8_t* buffer, uint64_t bufferSize, uint64_t offset); public: /** @@ -340,6 +367,7 @@ namespace Azure { namespace Core { namespace Http { { this->m_pCurl = curl_easy_init(); this->m_bodyStartInBuffer = 0; + this->m_innerBufferSize = LibcurlReaderSize; } /** @@ -412,6 +440,8 @@ namespace Azure { namespace Core { namespace Http { */ uint64_t m_offset; + bool m_unknownSize; + public: /** * @brief Construct a new Curl Body Stream object. @@ -425,12 +455,9 @@ namespace Azure { namespace Core { namespace Http { { } - ~CurlBodyStream() + CurlBodyStream(CurlSession* curlSession) + : m_length(0), m_curlSession(curlSession), m_offset(0), m_unknownSize(true) { - if (this->m_curlSession != nullptr) - { - delete this->m_curlSession; // Session Destructor will cleanup libcurl handle - } } /** @@ -450,13 +477,14 @@ namespace Azure { namespace Core { namespace Http { */ uint64_t Read(uint8_t* buffer, uint64_t count) override { - if (this->m_length == this->m_offset) + if (this->m_length == this->m_offset && !this->m_unknownSize) { return 0; } // Read bytes from curl into buffer. As max as the length of Stream is allowed auto readCount = this->m_curlSession->ReadWithOffset(buffer, count, this->m_offset); this->m_offset += readCount; + return readCount; } diff --git a/sdk/core/azure-core/inc/http/http.hpp b/sdk/core/azure-core/inc/http/http.hpp index d26e313821..4e97f83547 100644 --- a/sdk/core/azure-core/inc/http/http.hpp +++ b/sdk/core/azure-core/inc/http/http.hpp @@ -215,7 +215,7 @@ namespace Azure { namespace Core { namespace Http { std::map m_retryHeaders; std::map m_retryQueryParameters; // Work only with streams - BodyStream* m_bodyStream; + std::unique_ptr m_bodyStream; // flag to know where to insert header bool m_retryModeEnabled; @@ -233,25 +233,14 @@ namespace Azure { namespace Core { namespace Http { std::string GetQueryString() const; public: - Request(HttpMethod httpMethod, std::string const& url, BodyStream* bodyStream) - : m_method(std::move(httpMethod)), m_url(url), m_bodyStream(bodyStream), + Request(HttpMethod httpMethod, std::string const& url, std::unique_ptr bodyStream) + : m_method(std::move(httpMethod)), m_url(url), m_bodyStream(std::move(bodyStream)), m_retryModeEnabled(false) { } // Typically used for GET with no request body. - Request(HttpMethod httpMethod, std::string const& url) - : Request(httpMethod, url, BodyStream::null) - { - } - - ~Request() - { - if (this->m_bodyStream != BodyStream::null) - { - delete this->m_bodyStream; - } - } + Request(HttpMethod httpMethod, std::string const& url) : Request(httpMethod, url, nullptr) {} // Methods used to build HTTP request void AppendPath(std::string const& path); @@ -296,16 +285,16 @@ namespace Azure { namespace Core { namespace Http { std::string m_reasonPhrase; std::map m_headers; - BodyStream* m_bodyStream; + std::unique_ptr m_bodyStream; Response( int32_t majorVersion, int32_t minorVersion, HttpStatusCode statusCode, std::string const& reasonPhrase, - BodyStream* const BodyStream) + std::unique_ptr BodyStream) : m_majorVersion(majorVersion), m_minorVersion(minorVersion), m_statusCode(statusCode), - m_reasonPhrase(reasonPhrase), m_bodyStream(BodyStream) + m_reasonPhrase(reasonPhrase), m_bodyStream(std::move(BodyStream)) { } @@ -315,23 +304,15 @@ namespace Azure { namespace Core { namespace Http { int32_t minorVersion, HttpStatusCode statusCode, std::string const& reasonPhrase) - : Response(majorVersion, minorVersion, statusCode, reasonPhrase, BodyStream::null) - { - } - - ~Response() + : Response(majorVersion, minorVersion, statusCode, reasonPhrase, nullptr) { - if (this->m_bodyStream != BodyStream::null) - { - delete this->m_bodyStream; - } } // Methods used to build HTTP response void AddHeader(std::string const& name, std::string const& value); // rfc form header-name: OWS header-value OWS void AddHeader(std::string const& header); - void SetBodyStream(BodyStream* stream); + void SetBodyStream(std::unique_ptr stream); // adding getters for version and stream body. Clang will complain on Mac if we have unused // fields in a class @@ -340,7 +321,15 @@ namespace Azure { namespace Core { namespace Http { HttpStatusCode GetStatusCode() const; std::string const& GetReasonPhrase(); std::map const& GetHeaders(); - BodyStream* GetBodyStream() { return this->m_bodyStream; } + std::unique_ptr GetBodyStream() + { + if (this->m_bodyStream == nullptr) + { + // Moved before or not yet created + return nullptr; + } + return std::move(this->m_bodyStream); + } // Allocates a buffer in heap and reads and copy stream content into it. // util for any API that needs to get the content from stream as a buffer diff --git a/sdk/core/azure-core/inc/http/stream.hpp b/sdk/core/azure-core/inc/http/stream.hpp index 4e8bae4578..4cc2448141 100644 --- a/sdk/core/azure-core/inc/http/stream.hpp +++ b/sdk/core/azure-core/inc/http/stream.hpp @@ -14,7 +14,7 @@ namespace Azure { namespace Core { namespace Http { // BodyStream is used to read data to/from a service class BodyStream { public: - static BodyStream* null; + virtual ~BodyStream() = 0; // Returns the length of the data; used with the HTTP Content-Length header virtual uint64_t Length() const = 0; @@ -33,9 +33,6 @@ namespace Azure { namespace Core { namespace Http { // Closes the stream; typically called after all data read or if an error occurs. virtual void Close() = 0; - - // Desstructor. Enables derived classes to call its destructor - virtual ~BodyStream() = 0; }; class MemoryBodyStream : public BodyStream { diff --git a/sdk/core/azure-core/src/credentials/credentials.cpp b/sdk/core/azure-core/src/credentials/credentials.cpp index 44ca32944d..7fc2db270a 100644 --- a/sdk/core/azure-core/src/credentials/credentials.cpp +++ b/sdk/core/azure-core/src/credentials/credentials.cpp @@ -73,7 +73,7 @@ AccessToken ClientSecretCredential::GetToken( auto bodyStream = std::make_unique(bodyVec); - Http::Request request(Http::HttpMethod::Post, url.str(), bodyStream.get()); + Http::Request request(Http::HttpMethod::Post, url.str(), std::move(bodyStream)); bodyStream.release(); request.AddHeader("Content-Type", "application/x-www-form-urlencoded"); @@ -103,8 +103,8 @@ AccessToken ClientSecretCredential::GetToken( { std::ostringstream errorMsg; errorMsg << errorMsgPrefix << "error response: " - << static_cast::type>(statusCode) - << " " << response->GetReasonPhrase(); + << static_cast::type>(statusCode) << " " + << response->GetReasonPhrase(); throw AuthenticationException(errorMsg.str()); } diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 230a811881..5e76354617 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -215,11 +215,11 @@ CURLcode CurlSession::HttpRawSend() // Send body 64k at a time (libcurl default) // NOTE: if stream is on top a contiguous memory, we can avoid allocating this copying buffer - std::unique_ptr unique_buffer(new uint8_t[UploadSstreamPageSize]); + std::unique_ptr unique_buffer(new uint8_t[UploadStreamPageSize]); auto buffer = unique_buffer.get(); while (rawRequestLen > 0) { - rawRequestLen = streamBody->Read(buffer, UploadSstreamPageSize); + rawRequestLen = streamBody->Read(buffer, UploadStreamPageSize); sendResult = SendBuffer(buffer, (size_t)rawRequestLen); } return sendResult; @@ -229,44 +229,195 @@ CURLcode CurlSession::HttpRawSend() CURLcode CurlSession::ReadStatusLineAndHeadersFromRawResponse() { auto parser = ResponseBufferParser(); + auto bufferSize = uint64_t(); + // Select a default reading strategy. // No content-length or Transfer-Encoding + this->m_bodyLengthType = ResponseBodyLengthType::ReadToCloseConnection; // Keep reading until all headers were read while (!parser.IsParseCompleted()) { // Try to fill internal buffer from socket. // If response is smaller than buffer, we will get back the size of the response - auto bufferSize = ReadSocketToBuffer(this->m_readBuffer, LibcurlReaderSize); + bufferSize = ReadSocketToBuffer(this->m_readBuffer, LibcurlReaderSize); // parse from buffer to create response auto bytesParsed = parser.Parse(this->m_readBuffer, (size_t)bufferSize); // if end of headers is reach before the end of response, that's where body start - if (bytesParsed < bufferSize) + if (bytesParsed + 2 < bufferSize) { - this->m_bodyStartInBuffer = bytesParsed + 1; // Set the start of body (skip \n) + this->m_bodyStartInBuffer = bytesParsed + 1; // Set the start of body (skip \r) } } this->m_response = parser.GetResponse(); + this->m_innerBufferSize = static_cast(bufferSize); // For Head request, set the length of body response to 0. if (this->m_request.GetMethod() == HttpMethod::Head) { - this->m_response->SetBodyStream(new CurlBodyStream(0, this)); + this->m_bodyLengthType = ResponseBodyLengthType::NoBody; + this->m_response->SetBodyStream(std::make_unique(0, this)); return CURLE_OK; } // TODO: tolower ContentLength auto headers = this->m_response->GetHeaders(); - auto bodySize = atoi(headers.at("Content-Length").data()); - // content-length is used later by session and session won't have access to the response any more - // (unique_ptr), so we save this value - this->m_contentLength = bodySize; - // Move session to live inside the stream from response. - this->m_response->SetBodyStream(new CurlBodyStream(bodySize, this)); + auto isContentLengthHeaderInResponse = headers.find("Content-Length"); + if (isContentLengthHeaderInResponse != headers.end()) + { + // Response with Content-Length + auto bodySize = std::stoull(headers.at("Content-Length").data()); + // content-length is used later by session and session won't have access to the response any + // more (unique_ptr), so we save this value + this->m_contentLength = bodySize; + this->m_bodyLengthType = ResponseBodyLengthType::ContentLength; + // Move session to live inside the stream from response. + this->m_response->SetBodyStream(std::make_unique(bodySize, this)); + return CURLE_OK; + } + + auto isTransferEncodingHeaderInResponse = headers.find("Transfer-Encoding"); + if (isTransferEncodingHeaderInResponse != headers.end()) + { + auto headerValue = isTransferEncodingHeaderInResponse->second; + auto isChunked = headerValue.find("chunked"); + + if (isChunked != std::string::npos) + { + // set curl session to know response is chunked + // This will be used to remove chunked info while reading + this->m_bodyLengthType = ResponseBodyLengthType::Chunked; + } + } + + /* + https://tools.ietf.org/html/rfc7230#section-3.3.3 + 7. Otherwise, this is a response message without a declared message + body length, so the message body length is determined by the + number of octets received prior to the server closing the + connection. + */ + + // Use unknown size CurlBodyStream. CurlSession will use the ResponseBodyLengthType to select a + // reading strategy + this->m_response->SetBodyStream(std::make_unique(this)); return CURLE_OK; } +uint64_t CurlSession::ReadChunkedBody(uint8_t* buffer, uint64_t bufferSize, uint64_t offset) +{ + // Remove the chunk info up to the next delimiter \r\n + if (offset == 0) + { + // first time calling read. move to the next \r + if (this->m_bodyStartInBuffer > 0 + && this->m_bodyStartInBuffer + offset < this->m_innerBufferSize) + { + for (uint64_t index = 1; index < this->m_innerBufferSize - this->m_bodyStartInBuffer; index++) + { + if (this->m_readBuffer[this->m_bodyStartInBuffer + index] == '\r') + { + // found end of chunked info. Start reading from there + return ReadChunkedBody(buffer, bufferSize, offset + index + 1); // +1 to skip found '\r' + } + } + // Inner buffer only has part or chunked info. Set it as no body in it + // Then read again + this->m_bodyStartInBuffer = 0; + return ReadChunkedBody(buffer, bufferSize, offset); + } + else + { + // nothing on internal buffer, and first read. Let's read from socket until we found \r + auto totalRead = uint64_t(); + while (ReadSocketToBuffer(buffer, 1) != 0) + { + totalRead += 1; + if (buffer[0] == '\r') + { + return ReadChunkedBody(buffer, bufferSize, offset + totalRead); + } + } + // Didn't fin the end of chunked data in body. throw + throw; + } + } + + uint64_t totalOffset = this->m_bodyStartInBuffer + offset; + auto writePosition = buffer; + auto toBeWritten = bufferSize; + auto bytesRead = uint64_t(); + + // At this point, offset must be greater than 0, and we are after \r. We must read \n next and + // then the body + if (this->m_bodyStartInBuffer > 0 && totalOffset < this->m_innerBufferSize) + { + if (this->m_readBuffer[totalOffset] == '\n') + { + // increase offset and advance to next position + return ReadChunkedBody(buffer, bufferSize, offset + 1); + } + + // Check if the end of chunked body is at inner buffer + auto endOfChunkedBody = std::find( + this->m_readBuffer + totalOffset, this->m_readBuffer + this->m_innerBufferSize, '\r'); + + if (endOfChunkedBody != this->m_readBuffer + this->m_innerBufferSize) + { + // reduce the size of the body to the end of body. This way trying to read more than the body + // end will end up reading up to the body end only + this->m_innerBufferSize + = std::distance(this->m_readBuffer + this->m_innerBufferSize, endOfChunkedBody); + toBeWritten = 0; // Setting to zero to avoid reading from buffer + } + + // Still have some body content in internal buffer after skipping \n + if (bufferSize < this->m_innerBufferSize - totalOffset) + { + // requested less content than available in internal buffer + std::memcpy(buffer, this->m_readBuffer + totalOffset, (size_t)bufferSize); + return bufferSize; + } + + // requested more than what it's available in internal buffer + bytesRead = this->m_innerBufferSize - totalOffset; + std::memcpy(buffer, this->m_readBuffer + totalOffset, (size_t)bytesRead + 1); + writePosition += bytesRead; + // setting toBeWritten + if (toBeWritten > 0) + { + toBeWritten -= bytesRead; + } + } + + if (toBeWritten > 0) + { + // Read from socket + bytesRead += ReadSocketToBuffer(writePosition, (size_t)toBeWritten); + if (bytesRead > 0) + { + // Check if reading include chunked termination and remove it if true + auto endOfBody = std::find(buffer, buffer + bytesRead, '\r'); + if (endOfBody != buffer + bytesRead) + { + // Read all remaining to close connection + { + constexpr uint64_t finalRead = 50; // usually only 5 more bytes are gotten "0\r\n\r\n" + uint8_t b[finalRead]; + ReadSocketToBuffer(b, finalRead); + curl_easy_cleanup(this->m_pCurl); + } + return bytesRead - std::distance(endOfBody, buffer + bytesRead) + 1; + } + return bytesRead; // didn't find end of body + } + } + + // Return read bytes + return 0; +} + uint64_t CurlSession::ReadWithOffset(uint8_t* buffer, uint64_t bufferSize, uint64_t offset) { if (bufferSize <= 0) @@ -274,6 +425,12 @@ uint64_t CurlSession::ReadWithOffset(uint8_t* buffer, uint64_t bufferSize, uint6 return 0; } + if (this->m_bodyLengthType == ResponseBodyLengthType::Chunked) + { + // won't use content-length as the maximun to be read + return ReadChunkedBody(buffer, bufferSize, offset); + } + // calculate where to start reading from inner buffer auto fixedOffset = offset == 0 ? offset + 1 : offset; // advance the last '\n' from headers end on first read @@ -291,14 +448,14 @@ uint64_t CurlSession::ReadWithOffset(uint8_t* buffer, uint64_t bufferSize, uint6 // If bodyStartInBuffer is set and while innerBufferStart is less than the buffer, it means there // is still data at innerbuffer for the body that is not yet read - if (this->m_bodyStartInBuffer > 0 && LibcurlReaderSize > innerBufferStart) + if (this->m_bodyStartInBuffer > 0 && this->m_innerBufferSize > innerBufferStart) { // Calculate the right size of innerBuffer: // It can be smaller than the total body, in that case we will take as much as the size of // buffer // It can be bugger than the total body, in that case we will take as much as the size of the // body - auto innerBufferWithBodyContent = LibcurlReaderSize - innerBufferStart; + auto innerBufferWithBodyContent = this->m_innerBufferSize - innerBufferStart; auto innerbufferSize = remainingBodySize < innerBufferWithBodyContent ? remainingBodySize : innerBufferWithBodyContent; @@ -338,40 +495,24 @@ uint64_t CurlSession::ReadWithOffset(uint8_t* buffer, uint64_t bufferSize, uint6 return bytesRead; } -// Read from socket until buffer is full or until socket has no more data +// Read from socket and return the number of bytes taken from socket uint64_t CurlSession::ReadSocketToBuffer(uint8_t* buffer, size_t bufferSize) { CURLcode readResult; size_t readBytes = 0; - size_t totalRead = 0; - auto pendingToRead = bufferSize; - while (!this->m_rawResponseEOF && pendingToRead > 0) + do // try to read from socket until response is OK { - do // try to read from socket until response is OK - { - readResult = curl_easy_recv(this->m_pCurl, buffer + totalRead, pendingToRead, &readBytes); + readResult = curl_easy_recv(this->m_pCurl, buffer, bufferSize, &readBytes); - // socket not ready. Wait or fail on timeout - if (readResult == CURLE_AGAIN && !WaitForSocketReady(this->m_curlSocket, 1, 60000L)) - { - throw; - } - } while (readResult == CURLE_AGAIN); // Keep trying to read until result is not CURLE_AGAIN - - // At this point we read, update counters - totalRead += readBytes; - pendingToRead -= readBytes; - - if (readBytes == 0) + // socket not ready. Wait or fail on timeout + if (readResult == CURLE_AGAIN && !WaitForSocketReady(this->m_curlSocket, 1, 60000L)) { - // set socket as nothing more to read - this->m_rawResponseEOF = true; - break; + throw; } - } + } while (readResult == CURLE_AGAIN); // Keep trying to read until result is not CURLE_AGAIN - return totalRead; + return readBytes; } std::unique_ptr CurlSession::GetResponse() diff --git a/sdk/core/azure-core/src/http/http.cpp b/sdk/core/azure-core/src/http/http.cpp deleted file mode 100644 index a6c9abf57b..0000000000 --- a/sdk/core/azure-core/src/http/http.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: MIT - -#include - -using namespace Azure::Core::Http; - -BodyStream* BodyStream::null = nullptr; diff --git a/sdk/core/azure-core/src/http/request.cpp b/sdk/core/azure-core/src/http/request.cpp index 20689bd783..f4d2c44893 100644 --- a/sdk/core/azure-core/src/http/request.cpp +++ b/sdk/core/azure-core/src/http/request.cpp @@ -78,7 +78,17 @@ std::map Request::GetHeaders() const return Request::MergeMaps(this->m_retryHeaders, this->m_headers); } -BodyStream* Request::GetBodyStream() { return m_bodyStream; } +// Return a reference to the unique ptr. We don't move the ownership here out of response. +BodyStream* Request::GetBodyStream() +{ + if (this->m_bodyStream == nullptr) + { + // no body in request + return nullptr; + } + // retuning BodyStream* without removing the ownership from Request. + return m_bodyStream.get(); +} // Writes an HTTP request with RFC2730 without the body (head line and headers) // https://tools.ietf.org/html/rfc7230#section-3.1.1 diff --git a/sdk/core/azure-core/src/http/response.cpp b/sdk/core/azure-core/src/http/response.cpp index 85ce7e6485..71c21c0dea 100644 --- a/sdk/core/azure-core/src/http/response.cpp +++ b/sdk/core/azure-core/src/http/response.cpp @@ -47,7 +47,10 @@ void Response::AddHeader(std::string const& name, std::string const& value) this->m_headers.insert(std::pair(name, value)); } -void Response::SetBodyStream(BodyStream* stream) { this->m_bodyStream = stream; } +void Response::SetBodyStream(std::unique_ptr stream) +{ + this->m_bodyStream = std::move(stream); +} std::unique_ptr> Response::ConstructBodyBufferFromStream( BodyStream* const stream) diff --git a/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp index ca322e3b44..f0878f07e8 100644 --- a/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp +++ b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyBuffer.cpp @@ -87,7 +87,8 @@ Http::Request createGetRequest() string host("https://httpbin.org/get"); cout << "Creating a GET request to" << endl << "Host: " << host << endl; - auto request = Http::Request(Http::HttpMethod::Get, host, new Http::MemoryBodyStream(buffer)); + auto request + = Http::Request(Http::HttpMethod::Get, host, std::make_unique(buffer)); request.AddHeader("one", "header"); request.AddHeader("other", "header2"); request.AddHeader("header", "value"); @@ -112,7 +113,7 @@ Http::Request createPutRequest() buffer[BufferSize - 1] = '}'; // set buffer to look like a Json `{"x":"xxx...xxx"}` auto request - = Http::Request(Http::HttpMethod::Put, host, new Http::MemoryBodyStream(std::move(buffer))); + = Http::Request(Http::HttpMethod::Put, host, std::make_unique(buffer)); request.AddHeader("one", "header"); request.AddHeader("other", "header2"); request.AddHeader("header", "value"); @@ -142,8 +143,13 @@ void printRespose(std::unique_ptr response) cout << header.first << " : " << header.second << endl; } cout << "Body (buffer):" << endl; - auto responseBodyVector - = Http::Response::ConstructBodyBufferFromStream(response->GetBodyStream()); + auto bodyStream = response->GetBodyStream(); + if (bodyStream == nullptr) + { + // No body in response + return; + } + auto responseBodyVector = Http::Response::ConstructBodyBufferFromStream(bodyStream.get()); if (responseBodyVector != nullptr) { // print body only if response has a body. Head Response won't have body diff --git a/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp index 51619a85ee..4d68d4d0ae 100644 --- a/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp +++ b/sdk/samples/http_client/curl/src/azure_core_with_curl_bodyStream.cpp @@ -134,7 +134,8 @@ Http::Request createPutRequest() buffer[BufferSize - 2] = '\"'; buffer[BufferSize - 1] = '}'; // set buffer to look like a Json `{"x":"xxx...xxx"}` - auto request = Http::Request(Http::HttpMethod::Put, host, new Http::MemoryBodyStream(buffer)); + auto request + = Http::Request(Http::HttpMethod::Put, host, std::make_unique(buffer)); request.AddHeader("one", "header"); request.AddHeader("other", "header2"); request.AddHeader("header", "value"); @@ -159,8 +160,8 @@ Http::Request createPutStreamRequest() bufferStream[StreamSize - 2] = '\"'; bufferStream[StreamSize - 1] = '}'; // set buffer to look like a Json `{"1":"111...111"}` - auto request = Http::Request( - Http::HttpMethod::Put, host, new MemoryBodyStream(bufferStream.data(), StreamSize)); + auto request + = Http::Request(Http::HttpMethod::Put, host, std::make_unique(buffer)); request.AddHeader("one", "header"); request.AddHeader("other", "header2"); request.AddHeader("header", "value"); diff --git a/sdk/storage/inc/blobs/append_blob_client.hpp b/sdk/storage/inc/blobs/append_blob_client.hpp index 84da2c449a..5d5106a3cf 100644 --- a/sdk/storage/inc/blobs/append_blob_client.hpp +++ b/sdk/storage/inc/blobs/append_blob_client.hpp @@ -43,7 +43,7 @@ namespace Azure { namespace Storage { namespace Blobs { BlobContentInfo Create(const CreateAppendBlobOptions& options = CreateAppendBlobOptions()); BlobAppendInfo AppendBlock( - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, const AppendBlockOptions& options = AppendBlockOptions()); BlobAppendInfo AppendBlockFromUri( diff --git a/sdk/storage/inc/blobs/block_blob_client.hpp b/sdk/storage/inc/blobs/block_blob_client.hpp index a4aded6421..9061632688 100644 --- a/sdk/storage/inc/blobs/block_blob_client.hpp +++ b/sdk/storage/inc/blobs/block_blob_client.hpp @@ -5,7 +5,6 @@ #include "blob_options.hpp" #include "blobs/blob_client.hpp" -#include "common/memory_stream.hpp" #include "common/storage_credential.hpp" #include "internal/protocol/blob_rest_client.hpp" @@ -43,12 +42,12 @@ namespace Azure { namespace Storage { namespace Blobs { BlockBlobClient WithSnapshot(const std::string& snapshot) const; BlobContentInfo Upload( - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, const UploadBlobOptions& options = UploadBlobOptions()) const; BlockInfo StageBlock( const std::string& blockId, - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, const StageBlockOptions& options = StageBlockOptions()) const; BlockInfo StageBlockFromUri( diff --git a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp index b9cadbf029..c24f0239b6 100644 --- a/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp +++ b/sdk/storage/inc/blobs/internal/protocol/blob_rest_client.hpp @@ -936,7 +936,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::string Date; std::string Version; std::string ClientRequestId; - Azure::Core::Http::BodyStream* BodyStream = nullptr; + std::unique_ptr BodyStream; std::string ETag; std::string LastModified; std::string ContentRange; @@ -998,7 +998,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request ListBlobContainersConstructRequest( const std::string& url, - const ListBlobContainersOptions& options) + ListBlobContainersOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); @@ -1057,7 +1057,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const ListBlobContainersOptions& options) + ListBlobContainersOptions& options) { auto request = ListBlobContainersConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -1072,7 +1072,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request GetUserDelegationKeyConstructRequest( const std::string& url, - const GetUserDelegationKeyOptions& options) + GetUserDelegationKeyOptions& options) { XmlWriter writer; GetUserDelegationKeyOptionsToXml(writer, options); @@ -1082,7 +1082,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request( Azure::Core::Http::HttpMethod::Post, url, - new Azure::Core::Http::MemoryBodyStream(std::move(body_buffer))); + std::make_unique(std::move(body_buffer))); request.AddHeader("Content-Length", std::to_string(body_buffer_length)); request.AddQueryParameter("restype", "service"); request.AddQueryParameter("comp", "userdelegationkey"); @@ -1123,7 +1123,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const GetUserDelegationKeyOptions& options) + GetUserDelegationKeyOptions& options) { auto request = GetUserDelegationKeyConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -1577,7 +1577,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request CreateConstructRequest( const std::string& url, - const CreateOptions& options) + CreateOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -1633,7 +1633,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const CreateOptions& options) + CreateOptions& options) { auto request = CreateConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -1648,7 +1648,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request DeleteConstructRequest( const std::string& url, - const DeleteOptions& options) + DeleteOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Delete, url); request.AddHeader("Content-Length", "0"); @@ -1691,7 +1691,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const DeleteOptions& options) + DeleteOptions& options) { auto request = DeleteConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -1707,7 +1707,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request GetPropertiesConstructRequest( const std::string& url, - const GetPropertiesOptions& options) + GetPropertiesOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Head, url); request.AddHeader("Content-Length", "0"); @@ -1788,7 +1788,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const GetPropertiesOptions& options) + GetPropertiesOptions& options) { auto request = GetPropertiesConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -1803,7 +1803,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request SetMetadataConstructRequest( const std::string& url, - const SetMetadataOptions& options) + SetMetadataOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -1859,7 +1859,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const SetMetadataOptions& options) + SetMetadataOptions& options) { auto request = SetMetadataConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -1877,7 +1877,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request ListBlobsConstructRequest( const std::string& url, - const ListBlobsOptions& options) + ListBlobsOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); @@ -1947,7 +1947,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const ListBlobsOptions& options) + ListBlobsOptions& options) { auto request = ListBlobsConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -2416,7 +2416,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request DownloadConstructRequest( const std::string& url, - const DownloadOptions& options) + DownloadOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); @@ -2598,7 +2598,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const DownloadOptions& options) + DownloadOptions& options) { auto request = DownloadConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -2616,7 +2616,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request DeleteConstructRequest( const std::string& url, - const DeleteOptions& options) + DeleteOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Delete, url); request.AddHeader("Content-Length", "0"); @@ -2671,7 +2671,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const DeleteOptions& options) + DeleteOptions& options) { auto request = DeleteConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -2684,7 +2684,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request UndeleteConstructRequest( const std::string& url, - const UndeleteOptions& options) + UndeleteOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -2720,7 +2720,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const UndeleteOptions& options) + UndeleteOptions& options) { auto request = UndeleteConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -2737,7 +2737,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request GetPropertiesConstructRequest( const std::string& url, - const GetPropertiesOptions& options) + GetPropertiesOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Head, url); request.AddHeader("Content-Length", "0"); @@ -2895,7 +2895,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const GetPropertiesOptions& options) + GetPropertiesOptions& options) { auto request = GetPropertiesConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -2921,7 +2921,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request SetHttpHeadersConstructRequest( const std::string& url, - const SetHttpHeadersOptions& options) + SetHttpHeadersOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3016,7 +3016,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const SetHttpHeadersOptions& options) + SetHttpHeadersOptions& options) { auto request = SetHttpHeadersConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3037,7 +3037,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request SetMetadataConstructRequest( const std::string& url, - const SetMetadataOptions& options) + SetMetadataOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3116,7 +3116,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const SetMetadataOptions& options) + SetMetadataOptions& options) { auto request = SetMetadataConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3131,7 +3131,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request SetAccessTierConstructRequest( const std::string& url, - const SetAccessTierOptions& options) + SetAccessTierOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3172,7 +3172,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const SetAccessTierOptions& options) + SetAccessTierOptions& options) { auto request = SetAccessTierConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3199,7 +3199,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request StartCopyFromUriConstructRequest( const std::string& url, - const StartCopyFromUriOptions& options) + StartCopyFromUriOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3303,7 +3303,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const StartCopyFromUriOptions& options) + StartCopyFromUriOptions& options) { auto request = StartCopyFromUriConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3318,7 +3318,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request AbortCopyFromUriConstructRequest( const std::string& url, - const AbortCopyFromUriOptions& options) + AbortCopyFromUriOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3359,7 +3359,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const AbortCopyFromUriOptions& options) + AbortCopyFromUriOptions& options) { auto request = AbortCopyFromUriConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3381,7 +3381,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request CreateSnapshotConstructRequest( const std::string& url, - const CreateSnapshotOptions& options) + CreateSnapshotOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3478,7 +3478,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const CreateSnapshotOptions& options) + CreateSnapshotOptions& options) { auto request = CreateSnapshotConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3492,7 +3492,7 @@ namespace Azure { namespace Storage { namespace Blobs { public: struct UploadOptions { - Azure::Core::Http::BodyStream* BodyStream = nullptr; + std::unique_ptr BodyStream; std::string ContentMD5; std::string ContentCRC64; BlobHttpHeaders Properties; @@ -3510,11 +3510,11 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request UploadConstructRequest( const std::string& url, - const UploadOptions& options) + UploadOptions& options) { uint64_t body_stream_length = options.BodyStream->Length(); auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, options.BodyStream); + Azure::Core::Http::HttpMethod::Put, url, std::move(options.BodyStream)); request.AddHeader("Content-Length", std::to_string(body_stream_length)); request.AddHeader("x-ms-version", "2019-07-07"); if (!options.EncryptionKey.empty()) @@ -3654,7 +3654,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const UploadOptions& options) + UploadOptions& options) { auto request = UploadConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3663,7 +3663,7 @@ namespace Azure { namespace Storage { namespace Blobs { struct StageBlockOptions { - Azure::Core::Http::BodyStream* BodyStream = nullptr; + std::unique_ptr BodyStream; std::string BlockId; std::string ContentMD5; std::string ContentCRC64; @@ -3675,11 +3675,11 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request StageBlockConstructRequest( const std::string& url, - const StageBlockOptions& options) + StageBlockOptions& options) { uint64_t body_stream_length = options.BodyStream->Length(); auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, options.BodyStream); + Azure::Core::Http::HttpMethod::Put, url, std::move(options.BodyStream)); request.AddHeader("Content-Length", std::to_string(body_stream_length)); request.AddQueryParameter("comp", "block"); request.AddQueryParameter("blockid", options.BlockId); @@ -3759,7 +3759,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const StageBlockOptions& options) + StageBlockOptions& options) { auto request = StageBlockConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3785,7 +3785,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request StageBlockFromUriConstructRequest( const std::string& url, - const StageBlockFromUriOptions& options) + StageBlockFromUriOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -3903,7 +3903,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const StageBlockFromUriOptions& options) + StageBlockFromUriOptions& options) { auto request = StageBlockFromUriConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -3928,7 +3928,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request CommitBlockListConstructRequest( const std::string& url, - const CommitBlockListOptions& options) + CommitBlockListOptions& options) { XmlWriter writer; CommitBlockListOptionsToXml(writer, options); @@ -3938,7 +3938,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Azure::Core::Http::Request( Azure::Core::Http::HttpMethod::Put, url, - new Azure::Core::Http::MemoryBodyStream(std::move(body_buffer))); + std::make_unique(std::move(body_buffer))); request.AddHeader("Content-Length", std::to_string(body_buffer_length)); request.AddQueryParameter("comp", "blocklist"); request.AddHeader("x-ms-version", "2019-07-07"); @@ -4061,7 +4061,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const CommitBlockListOptions& options) + CommitBlockListOptions& options) { auto request = CommitBlockListConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -4079,7 +4079,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request GetBlockListConstructRequest( const std::string& url, - const GetBlockListOptions& options) + GetBlockListOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); @@ -4145,7 +4145,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const GetBlockListOptions& options) + GetBlockListOptions& options) { auto request = GetBlockListConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -4321,7 +4321,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request CreateConstructRequest( const std::string& url, - const CreateOptions& options) + CreateOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -4457,7 +4457,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const CreateOptions& options) + CreateOptions& options) { auto request = CreateConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -4466,7 +4466,7 @@ namespace Azure { namespace Storage { namespace Blobs { struct UploadPagesOptions { - Azure::Core::Http::BodyStream* BodyStream = nullptr; + std::unique_ptr BodyStream; std::pair Range; std::string ContentMD5; std::string ContentCRC64; @@ -4482,11 +4482,11 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request UploadPagesConstructRequest( const std::string& url, - const UploadPagesOptions& options) + UploadPagesOptions& options) { uint64_t body_stream_length = options.BodyStream->Length(); auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, options.BodyStream); + Azure::Core::Http::HttpMethod::Put, url, std::move(options.BodyStream)); request.AddHeader("Content-Length", std::to_string(body_stream_length)); request.AddQueryParameter("comp", "page"); request.AddHeader("x-ms-version", "2019-07-07"); @@ -4605,7 +4605,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const UploadPagesOptions& options) + UploadPagesOptions& options) { auto request = UploadPagesConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -4631,7 +4631,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request UploadPagesFromUriConstructRequest( const std::string& url, - const UploadPagesFromUriOptions& options) + UploadPagesFromUriOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -4772,7 +4772,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const UploadPagesFromUriOptions& options) + UploadPagesFromUriOptions& options) { auto request = UploadPagesFromUriConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -4794,7 +4794,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request ClearPagesConstructRequest( const std::string& url, - const ClearPagesOptions& options) + ClearPagesOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -4897,7 +4897,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const ClearPagesOptions& options) + ClearPagesOptions& options) { auto request = ClearPagesConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -4918,7 +4918,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request ResizeConstructRequest( const std::string& url, - const ResizeOptions& options) + ResizeOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -4990,7 +4990,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const ResizeOptions& options) + ResizeOptions& options) { auto request = ResizeConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -5011,7 +5011,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request GetPageRangesConstructRequest( const std::string& url, - const GetPageRangesOptions& options) + GetPageRangesOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Get, url); request.AddHeader("Content-Length", "0"); @@ -5099,7 +5099,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const GetPageRangesOptions& options) + GetPageRangesOptions& options) { auto request = GetPageRangesConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -5117,7 +5117,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request CopyIncrementalConstructRequest( const std::string& url, - const CopyIncrementalOptions& options) + CopyIncrementalOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -5174,7 +5174,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const CopyIncrementalOptions& options) + CopyIncrementalOptions& options) { auto request = CopyIncrementalConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -5366,7 +5366,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request CreateConstructRequest( const std::string& url, - const CreateOptions& options) + CreateOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -5495,7 +5495,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const CreateOptions& options) + CreateOptions& options) { auto request = CreateConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -5504,7 +5504,7 @@ namespace Azure { namespace Storage { namespace Blobs { struct AppendBlockOptions { - Azure::Core::Http::BodyStream* BodyStream = nullptr; + std::unique_ptr BodyStream; std::string ContentMD5; std::string ContentCRC64; std::string LeaseId; @@ -5521,11 +5521,11 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request AppendBlockConstructRequest( const std::string& url, - const AppendBlockOptions& options) + AppendBlockOptions& options) { uint64_t body_stream_length = options.BodyStream->Length(); auto request = Azure::Core::Http::Request( - Azure::Core::Http::HttpMethod::Put, url, options.BodyStream); + Azure::Core::Http::HttpMethod::Put, url, std::move(options.BodyStream)); request.AddHeader("Content-Length", std::to_string(body_stream_length)); request.AddQueryParameter("comp", "appendblock"); request.AddHeader("x-ms-version", "2019-07-07"); @@ -5635,7 +5635,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const AppendBlockOptions& options) + AppendBlockOptions& options) { auto request = AppendBlockConstructRequest(url, options); auto response = pipeline.Send(context, request); @@ -5662,7 +5662,7 @@ namespace Azure { namespace Storage { namespace Blobs { static Azure::Core::Http::Request AppendBlockFromUriConstructRequest( const std::string& url, - const AppendBlockFromUriOptions& options) + AppendBlockFromUriOptions& options) { auto request = Azure::Core::Http::Request(Azure::Core::Http::HttpMethod::Put, url); request.AddHeader("Content-Length", "0"); @@ -5795,7 +5795,7 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context context, Azure::Core::Http::HttpPipeline& pipeline, const std::string& url, - const AppendBlockFromUriOptions& options) + AppendBlockFromUriOptions& options) { auto request = AppendBlockFromUriConstructRequest(url, options); auto response = pipeline.Send(context, request); diff --git a/sdk/storage/inc/blobs/page_blob_client.hpp b/sdk/storage/inc/blobs/page_blob_client.hpp index 9aadc4c695..96d153419d 100644 --- a/sdk/storage/inc/blobs/page_blob_client.hpp +++ b/sdk/storage/inc/blobs/page_blob_client.hpp @@ -64,7 +64,7 @@ namespace Azure { namespace Storage { namespace Blobs { const CreatePageBlobOptions& options = CreatePageBlobOptions()); PageInfo UploadPages( - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, uint64_t offset, const UploadPagesOptions& options = UploadPagesOptions()); diff --git a/sdk/storage/inc/common/memory_stream.hpp b/sdk/storage/inc/common/memory_stream.hpp index 67cdb2923d..472c07aa43 100644 --- a/sdk/storage/inc/common/memory_stream.hpp +++ b/sdk/storage/inc/common/memory_stream.hpp @@ -5,13 +5,16 @@ #include "http/stream.hpp" +#include #include namespace Azure { namespace Storage { class MemoryStream : public Azure::Core::Http::BodyStream { public: - explicit MemoryStream(const uint8_t* data, std::size_t length) : m_data(data), m_length(length) {} + explicit MemoryStream(const uint8_t* data, std::size_t length) : m_data(data), m_length(length) + { + } ~MemoryStream() override {} @@ -29,4 +32,9 @@ namespace Azure { namespace Storage { std::size_t m_offset = 0; }; + inline std::unique_ptr CreateMemoryStream(const uint8_t* data, std::size_t length) + { + return std::make_unique(data, length); + } + }} // namespace Azure::Storage diff --git a/sdk/storage/inc/datalake/path_client.hpp b/sdk/storage/inc/datalake/path_client.hpp index 2fbc76ec4e..5e2725ef63 100644 --- a/sdk/storage/inc/datalake/path_client.hpp +++ b/sdk/storage/inc/datalake/path_client.hpp @@ -30,7 +30,7 @@ namespace Azure { namespace Storage { namespace DataLake { struct ReadPathResponse { - Azure::Core::Http::BodyStream* Body; + std::unique_ptr Body; std::string AcceptRanges; std::string CacheControl; std::string ContentDisposition; @@ -146,7 +146,7 @@ namespace Azure { namespace Storage { namespace DataLake { * @return PathAppendDataResponse */ PathAppendDataResponse AppendData( - Azure::Core::Http::BodyStream* stream, + std::unique_ptr stream, int64_t offset, const PathAppendDataOptions& options = PathAppendDataOptions()) const; diff --git a/sdk/storage/inc/datalake/protocol/datalake_rest_client.hpp b/sdk/storage/inc/datalake/protocol/datalake_rest_client.hpp index 1cbc8d2ebd..090e96fc8d 100644 --- a/sdk/storage/inc/datalake/protocol/datalake_rest_client.hpp +++ b/sdk/storage/inc/datalake/protocol/datalake_rest_client.hpp @@ -637,7 +637,7 @@ namespace Azure { namespace Storage { namespace DataLake { struct PathReadResponse { - Azure::Core::Http::BodyStream* BodyStream; + std::unique_ptr BodyStream; std::string AcceptRanges; std::string CacheControl; std::string ContentDisposition; @@ -833,7 +833,7 @@ namespace Azure { namespace Storage { namespace DataLake { = ServiceListFileSystemsResponse::ServiceListFileSystemsResponseFromFileSystemList( FileSystemList::CreateFromJson(nlohmann::json::parse( *Azure::Core::Http::Response::ConstructBodyBufferFromStream( - response.GetBodyStream())))); + response.GetBodyStream().get())))); if (response.GetHeaders().find(Details::c_HeaderDate) != response.GetHeaders().end()) { result.Date = response.GetHeaders().at(Details::c_HeaderDate); @@ -1349,7 +1349,7 @@ namespace Azure { namespace Storage { namespace DataLake { = FileSystemListPathsResponse::FileSystemListPathsResponseFromPathList( PathList::CreateFromJson(nlohmann::json::parse( *Azure::Core::Http::Response::ConstructBodyBufferFromStream( - response.GetBodyStream())))); + response.GetBodyStream().get())))); if (response.GetHeaders().find(Details::c_HeaderDate) != response.GetHeaders().end()) { result.Date = response.GetHeaders().at(Details::c_HeaderDate); @@ -1614,16 +1614,16 @@ namespace Azure { namespace Storage { namespace DataLake { // has been modified since the specified date/time. std::string IfUnmodifiedSince; // Specify this header value to operate only on a blob if it // has not been modified since the specified date/time. - Azure::Core::Http::BodyStream* Body; // The stream that contains the body of this request. }; static PathUpdateResponse Update( std::string url, Azure::Core::Http::HttpPipeline& pipeline, Azure::Core::Context context, + std::unique_ptr content, const UpdateOptions& updateOptions) { - auto request = UpdateCreateRequest(std::move(url), updateOptions); + auto request = UpdateCreateRequest(std::move(url), std::move(content), updateOptions); return UpdateParseResponse(pipeline.Send(context, request)); } @@ -2024,7 +2024,7 @@ namespace Azure { namespace Storage { namespace DataLake { // validated by the service. std::string LeaseIdOptional; // If specified, the operation only succeeds if the resource's // lease is active and matches this ID. - Azure::Core::Http::BodyStream* Body; // The stream that contains the body of this request. + std::string ClientRequestId; // Provides a client-generated, opaque value with a 1 KB // character limit that is recorded in the analytics logs when // storage analytics logging is enabled. @@ -2037,9 +2037,11 @@ namespace Azure { namespace Storage { namespace DataLake { std::string url, Azure::Core::Http::HttpPipeline& pipeline, Azure::Core::Context context, + std::unique_ptr content, const AppendDataOptions& appendDataOptions) { - auto request = AppendDataCreateRequest(std::move(url), appendDataOptions); + auto request + = AppendDataCreateRequest(std::move(url), std::move(content), appendDataOptions); return AppendDataParseResponse(pipeline.Send(context, request)); } @@ -2203,10 +2205,11 @@ namespace Azure { namespace Storage { namespace DataLake { static Azure::Core::Http::Request UpdateCreateRequest( std::string url, + std::unique_ptr content, const UpdateOptions& updateOptions) { Azure::Core::Http::Request request( - Azure::Core::Http::HttpMethod::Patch, std::move(url), updateOptions.Body); + Azure::Core::Http::HttpMethod::Patch, std::move(url), std::move(content)); if (!updateOptions.ClientRequestId.empty()) { request.AddHeader(Details::c_HeaderClientRequestId, updateOptions.ClientRequestId); @@ -2321,7 +2324,7 @@ namespace Azure { namespace Storage { namespace DataLake { = PathUpdateResponse::PathUpdateResponseFromSetAccessControlRecursiveResponse( SetAccessControlRecursiveResponse::CreateFromJson(nlohmann::json::parse( *Azure::Core::Http::Response::ConstructBodyBufferFromStream( - response.GetBodyStream())))); + response.GetBodyStream().get())))); if (response.GetHeaders().find(Details::c_HeaderDate) != response.GetHeaders().end()) { result.Date = response.GetHeaders().at(Details::c_HeaderDate); @@ -3261,7 +3264,7 @@ namespace Azure { namespace Storage { namespace DataLake { PathSetAccessControlRecursiveResponseFromSetAccessControlRecursiveResponse( SetAccessControlRecursiveResponse::CreateFromJson(nlohmann::json::parse( *Azure::Core::Http::Response::ConstructBodyBufferFromStream( - response.GetBodyStream())))); + response.GetBodyStream().get())))); if (response.GetHeaders().find(Details::c_HeaderDate) != response.GetHeaders().end()) { result.Date = response.GetHeaders().at(Details::c_HeaderDate); @@ -3427,10 +3430,11 @@ namespace Azure { namespace Storage { namespace DataLake { static Azure::Core::Http::Request AppendDataCreateRequest( std::string url, + std::unique_ptr content, const AppendDataOptions& appendDataOptions) { Azure::Core::Http::Request request( - Azure::Core::Http::HttpMethod::Patch, std::move(url), appendDataOptions.Body); + Azure::Core::Http::HttpMethod::Patch, std::move(url), std::move(content)); request.AddQueryParameter(Details::c_QueryAction, "append"); // TODO: Need to check for Null when Nullable is ready diff --git a/sdk/storage/sample/blob_getting_started.cpp b/sdk/storage/sample/blob_getting_started.cpp index 6568f79eab..54b2bff7d2 100644 --- a/sdk/storage/sample/blob_getting_started.cpp +++ b/sdk/storage/sample/blob_getting_started.cpp @@ -29,21 +29,30 @@ void BlobsGettingStarted() BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); - auto blobContentStream = new Azure::Storage::MemoryStream( + auto blobContentStream = Azure::Storage::CreateMemoryStream( reinterpret_cast(blobContent.data()), blobContent.length()); - blobClient.Upload(blobContentStream); + blobClient.Upload(std::move(blobContentStream)); std::map blobMetadata = {{"key1", "value1"}, {"key2", "value2"}}; blobClient.SetMetadata(blobMetadata); auto blobDownloadContent = blobClient.Download(); blobContent.resize(static_cast(blobDownloadContent.BodyStream->Length())); - blobDownloadContent.BodyStream->Read(reinterpret_cast(&blobContent[0]), blobContent.length()); + // read body stream until it returns 0 (all content read) + auto readTotal = uint64_t(); + do + { + auto offset = uint64_t(); + readTotal = blobDownloadContent.BodyStream->Read( + reinterpret_cast(&blobContent[(size_t)offset]), blobContent.length()); + offset += readTotal; + } while (readTotal != 0); + std::cout << blobContent << std::endl; auto properties = blobClient.GetProperties(); for (auto metadata : properties.Metadata) { - std::cout << metadata.first << ":" << metadata.second << std::endl; + std::cout << metadata.first << ":" << metadata.second << std::endl; } } diff --git a/sdk/storage/src/blobs/append_blob_client.cpp b/sdk/storage/src/blobs/append_blob_client.cpp index 1ee9faf7f8..1f91939065 100644 --- a/sdk/storage/src/blobs/append_blob_client.cpp +++ b/sdk/storage/src/blobs/append_blob_client.cpp @@ -71,11 +71,11 @@ namespace Azure { namespace Storage { namespace Blobs { } BlobAppendInfo AppendBlobClient::AppendBlock( - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, const AppendBlockOptions& options) { BlobRestClient::AppendBlob::AppendBlockOptions protocolLayerOptions; - protocolLayerOptions.BodyStream = content; + protocolLayerOptions.BodyStream = std::move(content); protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; protocolLayerOptions.LeaseId = options.LeaseId; diff --git a/sdk/storage/src/blobs/block_blob_client.cpp b/sdk/storage/src/blobs/block_blob_client.cpp index 4309416f2a..c5d1500510 100644 --- a/sdk/storage/src/blobs/block_blob_client.cpp +++ b/sdk/storage/src/blobs/block_blob_client.cpp @@ -58,11 +58,11 @@ namespace Azure { namespace Storage { namespace Blobs { } BlobContentInfo BlockBlobClient::Upload( - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, const UploadBlobOptions& options) const { BlobRestClient::BlockBlob::UploadOptions protocolLayerOptions; - protocolLayerOptions.BodyStream = content; + protocolLayerOptions.BodyStream = std::move(content); protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; protocolLayerOptions.Properties = options.Properties; @@ -78,11 +78,11 @@ namespace Azure { namespace Storage { namespace Blobs { BlockInfo BlockBlobClient::StageBlock( const std::string& blockId, - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, const StageBlockOptions& options) const { BlobRestClient::BlockBlob::StageBlockOptions protocolLayerOptions; - protocolLayerOptions.BodyStream = content; + protocolLayerOptions.BodyStream = std::move(content); protocolLayerOptions.BlockId = blockId; protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; diff --git a/sdk/storage/src/blobs/page_blob_client.cpp b/sdk/storage/src/blobs/page_blob_client.cpp index c01a612671..fa005dd78f 100644 --- a/sdk/storage/src/blobs/page_blob_client.cpp +++ b/sdk/storage/src/blobs/page_blob_client.cpp @@ -74,13 +74,14 @@ namespace Azure { namespace Storage { namespace Blobs { } PageInfo PageBlobClient::UploadPages( - Azure::Core::Http::BodyStream* content, + std::unique_ptr content, uint64_t offset, const UploadPagesOptions& options) { BlobRestClient::PageBlob::UploadPagesOptions protocolLayerOptions; - protocolLayerOptions.BodyStream = content; - protocolLayerOptions.Range = std::make_pair(offset, offset + content->Length() - 1); + protocolLayerOptions.BodyStream = std::move(content); + protocolLayerOptions.Range + = std::make_pair(offset, offset + protocolLayerOptions.BodyStream->Length() - 1); protocolLayerOptions.ContentMD5 = options.ContentMD5; protocolLayerOptions.ContentCRC64 = options.ContentCRC64; protocolLayerOptions.LeaseId = options.LeaseId; diff --git a/sdk/storage/src/common/storage_error.cpp b/sdk/storage/src/common/storage_error.cpp index 1c6ebd458e..c3e24b8742 100644 --- a/sdk/storage/src/common/storage_error.cpp +++ b/sdk/storage/src/common/storage_error.cpp @@ -11,8 +11,12 @@ namespace Azure { namespace Storage { StorageError StorageError::CreateFromResponse( /* const */ std::unique_ptr response) { - auto bodyBuffer - = Azure::Core::Http::Response::ConstructBodyBufferFromStream(response->GetBodyStream()); + auto bodyStream = response->GetBodyStream(); + auto bodyBuffer = std::make_unique>(); + if (bodyStream != nullptr) + { + bodyBuffer = Azure::Core::Http::Response::ConstructBodyBufferFromStream(bodyStream.get()); + } auto httpStatusCode = response->GetStatusCode(); std::string reasonPhrase = response->GetReasonPhrase(); diff --git a/sdk/storage/src/datalake/path_client.cpp b/sdk/storage/src/datalake/path_client.cpp index b508fcf495..da43490b2f 100644 --- a/sdk/storage/src/datalake/path_client.cpp +++ b/sdk/storage/src/datalake/path_client.cpp @@ -175,20 +175,19 @@ namespace Azure { namespace Storage { namespace DataLake { } PathAppendDataResponse PathClient::AppendData( - Azure::Core::Http::BodyStream* stream, + std::unique_ptr stream, int64_t offset, const PathAppendDataOptions& options) const { DataLakeRestClient::Path::AppendDataOptions protocolLayerOptions; // TODO: Add null check here when Nullable is supported - protocolLayerOptions.Body = stream; protocolLayerOptions.Position = offset; protocolLayerOptions.ContentLength = stream->Length(); protocolLayerOptions.TransactionalContentMD5 = options.ContentMD5; protocolLayerOptions.LeaseIdOptional = options.LeaseId; protocolLayerOptions.Timeout = options.Timeout; return DataLakeRestClient::Path::AppendData( - m_dfsUri.ToString(), *m_pipeline, options.Context, protocolLayerOptions); + m_dfsUri.ToString(), *m_pipeline, options.Context, std::move(stream), protocolLayerOptions); } PathFlushDataResponse PathClient::FlushData(int64_t offset, const PathFlushDataOptions& options) @@ -269,7 +268,7 @@ namespace Azure { namespace Storage { namespace DataLake { protocolLayerOptions.Properties = Details::SerializeMetadata(options.Metadata); protocolLayerOptions.Timeout = options.Timeout; return DataLakeRestClient::Path::Update( - m_dfsUri.ToString(), *m_pipeline, options.Context, protocolLayerOptions); + m_dfsUri.ToString(), *m_pipeline, options.Context, nullptr, protocolLayerOptions); } PathCreateResponse PathClient::Create(const PathCreateOptions& options) const @@ -373,31 +372,30 @@ namespace Azure { namespace Storage { namespace DataLake { auto result = DataLakeRestClient::Path::GetProperties( m_dfsUri.ToString(), *m_pipeline, options.Context, protocolLayerOptions); auto range = GetOffsetLength(result.ContentRange); - return GetPathPropertiesResponse{ - std::move(result.AcceptRanges), - std::move(result.CacheControl), - std::move(result.ContentDisposition), - std::move(result.ContentEncoding), - std::move(result.ContentLanguage), - result.ContentLength, - range.first, - range.second, - std::move(result.ContentType), - std::move(result.ContentMD5), - std::move(result.Date), - std::move(result.ETag), - std::move(result.LastModified), - std::move(result.RequestId), - std::move(result.Version), - std::move(result.ResourceType), - std::move(result.Owner), - std::move(result.Group), - std::move(result.Permissions), - Acl::DeserializeAcls(result.ACL), - std::move(result.LeaseDuration), - std::move(result.LeaseState), - std::move(result.LeaseStatus), - Details::DeserializeMetadata(result.Properties)}; + return GetPathPropertiesResponse{std::move(result.AcceptRanges), + std::move(result.CacheControl), + std::move(result.ContentDisposition), + std::move(result.ContentEncoding), + std::move(result.ContentLanguage), + result.ContentLength, + range.first, + range.second, + std::move(result.ContentType), + std::move(result.ContentMD5), + std::move(result.Date), + std::move(result.ETag), + std::move(result.LastModified), + std::move(result.RequestId), + std::move(result.Version), + std::move(result.ResourceType), + std::move(result.Owner), + std::move(result.Group), + std::move(result.Permissions), + Acl::DeserializeAcls(result.ACL), + std::move(result.LeaseDuration), + std::move(result.LeaseState), + std::move(result.LeaseStatus), + Details::DeserializeMetadata(result.Properties)}; } // TODO: Remove or uncomment after finalized how to support lease. @@ -440,28 +438,27 @@ namespace Azure { namespace Storage { namespace DataLake { auto result = DataLakeRestClient::Path::Read( m_dfsUri.ToString(), *m_pipeline, options.Context, protocolLayerOptions); auto range = GetOffsetLength(result.ContentRange); - return ReadPathResponse{ - result.BodyStream, - std::move(result.AcceptRanges), - std::move(result.CacheControl), - std::move(result.ContentDisposition), - std::move(result.ContentEncoding), - std::move(result.ContentLanguage), - std::move(result.ContentLength), - range.first, - range.second, - std::move(result.ContentType), - std::move(result.ContentMD5), - std::move(result.Date), - std::move(result.ETag), - std::move(result.LastModified), - std::move(result.RequestId), - std::move(result.Version), - std::move(result.ResourceType), - std::move(result.LeaseDuration), - std::move(result.LeaseState), - std::move(result.LeaseStatus), - std::move(result.XMsContentMd5), - Details::DeserializeMetadata(result.Properties)}; + return ReadPathResponse{std::move(result.BodyStream), + std::move(result.AcceptRanges), + std::move(result.CacheControl), + std::move(result.ContentDisposition), + std::move(result.ContentEncoding), + std::move(result.ContentLanguage), + std::move(result.ContentLength), + range.first, + range.second, + std::move(result.ContentType), + std::move(result.ContentMD5), + std::move(result.Date), + std::move(result.ETag), + std::move(result.LastModified), + std::move(result.RequestId), + std::move(result.Version), + std::move(result.ResourceType), + std::move(result.LeaseDuration), + std::move(result.LeaseState), + std::move(result.LeaseStatus), + std::move(result.XMsContentMd5), + Details::DeserializeMetadata(result.Properties)}; } }}} // namespace Azure::Storage::DataLake diff --git a/sdk/storage/test/append_blob_client_test.cpp b/sdk/storage/test/append_blob_client_test.cpp index 41472e6485..48c6e68269 100644 --- a/sdk/storage/test/append_blob_client_test.cpp +++ b/sdk/storage/test/append_blob_client_test.cpp @@ -30,7 +30,7 @@ namespace Azure { namespace Storage { namespace Test { m_blobUploadOptions.Properties.ContentMD5 = ""; m_appendBlobClient->Create(m_blobUploadOptions); m_appendBlobClient->AppendBlock( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size())); + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size())); m_blobUploadOptions.Properties.ContentMD5 = m_appendBlobClient->GetProperties().ContentMD5; } @@ -47,7 +47,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(properties.ContentLength, 0); appendBlobClient.AppendBlock( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size())); + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size())); properties = appendBlobClient.GetProperties(); EXPECT_EQ(properties.CommittedBlockCount, 1); EXPECT_EQ(properties.ContentLength, m_blobContent.size()); @@ -56,22 +56,24 @@ namespace Azure { namespace Storage { namespace Test { options.AppendPosition = 1_MB; EXPECT_THROW( appendBlobClient.AppendBlock( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), options), + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), + options), std::runtime_error); options.AppendPosition = properties.ContentLength; appendBlobClient.AppendBlock( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), options); + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), options); properties = appendBlobClient.GetProperties(); options = Azure::Storage::Blobs::AppendBlockOptions(); options.MaxSize = properties.ContentLength + m_blobContent.size() - 1; EXPECT_THROW( appendBlobClient.AppendBlock( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), options), + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), + options), std::runtime_error); options.MaxSize = properties.ContentLength + m_blobContent.size(); appendBlobClient.AppendBlock( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), options); + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), options); // TODO: AppendBlockFromUri must be authorized with SAS, but we don't have SAS for now. diff --git a/sdk/storage/test/blob_container_client_test.cpp b/sdk/storage/test/blob_container_client_test.cpp index 0f08ae7af8..f7e06a92b5 100644 --- a/sdk/storage/test/blob_container_client_test.cpp +++ b/sdk/storage/test/blob_container_client_test.cpp @@ -85,7 +85,7 @@ namespace Azure { namespace Storage { namespace Test { { std::string blobName = prefix1 + baseName + std::to_string(i); auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); - blobClient.Upload(new Azure::Storage::MemoryStream(nullptr, 0)); + blobClient.Upload(Azure::Storage::CreateMemoryStream(nullptr, 0)); p1Blobs.insert(blobName); p1p2Blobs.insert(blobName); } @@ -93,7 +93,7 @@ namespace Azure { namespace Storage { namespace Test { { std::string blobName = prefix2 + baseName + std::to_string(i); auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); - blobClient.Upload(new Azure::Storage::MemoryStream(nullptr, 0)); + blobClient.Upload(Azure::Storage::CreateMemoryStream(nullptr, 0)); p2Blobs.insert(blobName); p1p2Blobs.insert(blobName); } @@ -151,7 +151,7 @@ namespace Azure { namespace Storage { namespace Test { { blobName = blobName + delimiter + RandomString(); auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); - blobClient.Upload(new Azure::Storage::MemoryStream(nullptr, 0)); + blobClient.Upload(Azure::Storage::CreateMemoryStream(nullptr, 0)); blobs.insert(blobName); } diff --git a/sdk/storage/test/block_blob_client_test.cpp b/sdk/storage/test/block_blob_client_test.cpp index 4373aad51c..56e9c1d181 100644 --- a/sdk/storage/test/block_blob_client_test.cpp +++ b/sdk/storage/test/block_blob_client_test.cpp @@ -34,7 +34,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, m_blobName); m_blockBlobClient = std::make_shared(std::move(blockBlobClient)); - m_blobContent.resize(8_MB); + m_blobContent.resize((size_t)8_MB); RandomBuffer(reinterpret_cast(&m_blobContent[0]), m_blobContent.size()); m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}}; m_blobUploadOptions.Properties.ContentType = "application/x-binary"; @@ -45,7 +45,7 @@ namespace Azure { namespace Storage { namespace Test { m_blobUploadOptions.Properties.ContentMD5 = ""; m_blobUploadOptions.Tier = Azure::Storage::Blobs::AccessTier::Hot; m_blockBlobClient->Upload( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), m_blobUploadOptions); m_blobUploadOptions.Properties.ContentMD5 = m_blockBlobClient->GetProperties().ContentMD5; } @@ -57,7 +57,7 @@ namespace Azure { namespace Storage { namespace Test { auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); blockBlobClient.Upload( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), m_blobUploadOptions); blockBlobClient.Delete(); @@ -83,8 +83,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ( ReadBodyStream(res.BodyStream), std::vector( - m_blobContent.begin() + options.Offset, - m_blobContent.begin() + options.Offset + options.Length)); + m_blobContent.begin() + (size_t)options.Offset, + m_blobContent.begin() + (size_t)(options.Offset + options.Length))); EXPECT_FALSE(res.ContentRange.empty()); } @@ -117,7 +117,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(ReadBodyStream(snapshotClient.Download().BodyStream), m_blobContent); EXPECT_EQ(snapshotClient.GetProperties().Metadata, m_blobUploadOptions.Metadata); EXPECT_THROW( - snapshotClient.Upload(new Azure::Storage::MemoryStream(nullptr, 0)), std::runtime_error); + snapshotClient.Upload(Azure::Storage::CreateMemoryStream(nullptr, 0)), std::runtime_error); EXPECT_THROW(snapshotClient.SetMetadata({}), std::runtime_error); EXPECT_THROW( snapshotClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool), std::runtime_error); @@ -136,7 +136,7 @@ namespace Azure { namespace Storage { namespace Test { auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); blockBlobClient.Upload( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size())); + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size())); blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); blockBlobClient.SetAccessTier(Azure::Storage::Blobs::AccessTier::Cool); Azure::Storage::Blobs::SetBlobHttpHeadersOptions options; @@ -177,7 +177,7 @@ namespace Azure { namespace Storage { namespace Test { block1Content.resize(100); RandomBuffer(reinterpret_cast(&block1Content[0]), block1Content.size()); blockBlobClient.StageBlock( - blockId1, new Azure::Storage::MemoryStream(block1Content.data(), block1Content.size())); + blockId1, Azure::Storage::CreateMemoryStream(block1Content.data(), block1Content.size())); Azure::Storage::Blobs::CommitBlockListOptions options; options.Properties = m_blobUploadOptions.Properties; options.Metadata = m_blobUploadOptions.Metadata; diff --git a/sdk/storage/test/page_blob_client_test.cpp b/sdk/storage/test/page_blob_client_test.cpp index f792ba817a..bd2ac45b1a 100644 --- a/sdk/storage/test/page_blob_client_test.cpp +++ b/sdk/storage/test/page_blob_client_test.cpp @@ -19,7 +19,7 @@ namespace Azure { namespace Storage { namespace Test { StandardStorageConnectionString(), m_containerName, m_blobName); m_pageBlobClient = std::make_shared(std::move(pageBlobClient)); - m_blobContent.resize(1_KB); + m_blobContent.resize((size_t)1_KB); RandomBuffer(reinterpret_cast(&m_blobContent[0]), m_blobContent.size()); m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"KEY2", "Value2"}}; m_blobUploadOptions.Properties.ContentType = "application/x-binary"; @@ -30,7 +30,7 @@ namespace Azure { namespace Storage { namespace Test { m_blobUploadOptions.Properties.ContentMD5 = ""; m_pageBlobClient->Create(m_blobContent.size(), m_blobUploadOptions); m_pageBlobClient->UploadPages( - new Azure::Storage::MemoryStream(m_blobContent.data(), m_blobContent.size()), 0); + Azure::Storage::CreateMemoryStream(m_blobContent.data(), m_blobContent.size()), 0); m_blobUploadOptions.Properties.ContentMD5 = m_pageBlobClient->GetProperties().ContentMD5; } @@ -53,28 +53,30 @@ namespace Azure { namespace Storage { namespace Test { pageBlobClient.Create(0, m_blobUploadOptions); EXPECT_EQ(pageBlobClient.GetProperties().ContentLength, 0); - pageBlobClient.Resize(2_KB); + pageBlobClient.Resize((size_t)2_KB); EXPECT_EQ(pageBlobClient.GetProperties().ContentLength, 2_KB); - pageBlobClient.Resize(1_KB); + pageBlobClient.Resize((size_t)1_KB); EXPECT_EQ(pageBlobClient.GetProperties().ContentLength, 1_KB); } TEST_F(PageBlobClientTest, UploadClear) { std::vector blobContent; - blobContent.resize(4_KB); + blobContent.resize((size_t)4_KB); RandomBuffer(reinterpret_cast(&blobContent[0]), blobContent.size()); auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( StandardStorageConnectionString(), m_containerName, RandomString()); pageBlobClient.Create(8_KB, m_blobUploadOptions); - pageBlobClient.UploadPages(new Azure::Storage::MemoryStream(blobContent.data(), blobContent.size()), 2_KB); + pageBlobClient.UploadPages( + Azure::Storage::CreateMemoryStream(blobContent.data(), blobContent.size()), 2_KB); // |_|_|x|x| |x|x|_|_| - blobContent.insert(blobContent.begin(), 2_KB, '\x00'); - blobContent.resize(8_KB, '\x00'); + blobContent.insert(blobContent.begin(), (size_t)2_KB, '\x00'); + blobContent.resize((size_t)8_KB, '\x00'); pageBlobClient.ClearPages(2_KB, 1_KB); // |_|_|_|x| |x|x|_|_| - std::fill(blobContent.begin() + 2_KB, blobContent.begin() + 2_KB + 1_KB, '\x00'); + std::fill( + blobContent.begin() + (size_t)2_KB, blobContent.begin() + (size_t)(2_KB + 1_KB), '\x00'); auto downloadContent = pageBlobClient.Download(); EXPECT_EQ(ReadBodyStream(downloadContent.BodyStream), blobContent); @@ -96,8 +98,9 @@ namespace Azure { namespace Storage { namespace Test { auto snapshot = pageBlobClient.CreateSnapshot().Snapshot; // |_|_|_|x| |x|x|_|_| This is what's in snapshot - blobContent.resize(1_KB); - pageBlobClient.UploadPages(new Azure::Storage::MemoryStream(blobContent.data(), blobContent.size()), 0); + blobContent.resize((size_t)1_KB); + pageBlobClient.UploadPages( + Azure::Storage::CreateMemoryStream(blobContent.data(), blobContent.size()), 0); pageBlobClient.ClearPages(3_KB, 1_KB); // |x|_|_|_| |x|x|_|_| diff --git a/sdk/storage/test/test_base.cpp b/sdk/storage/test/test_base.cpp index fea748a987..460edf12fc 100644 --- a/sdk/storage/test/test_base.cpp +++ b/sdk/storage/test/test_base.cpp @@ -132,10 +132,27 @@ namespace Azure { namespace Storage { namespace Test { } } - std::vector ReadBodyStream(Azure::Core::Http::BodyStream* stream) + std::vector ReadBodyStream(std::unique_ptr& stream) { - std::vector body(stream->Length(), '\x00'); - stream->Read(&body[0], body.size()); + auto s = stream->Length(); + std::vector body((size_t)s, '\x00'); + // Read 15k at at time. warning C6262: exceeds /analyze:stacksize '16384'. + { + constexpr uint64_t fixedSize = 1024 * 15; + uint8_t tempBuffer[fixedSize]; + auto readBytes = uint64_t(); + auto offset = uint64_t(); + do + { + readBytes = stream->Read(tempBuffer, fixedSize); + for (uint64_t index = 0; index != readBytes; index++) + { + body[(size_t)(offset + index)] = tempBuffer[index]; + } + offset += readBytes; + } while (readBytes != 0); + } + return body; } diff --git a/sdk/storage/test/test_base.hpp b/sdk/storage/test/test_base.hpp index a2661d3099..acc77fe965 100644 --- a/sdk/storage/test/test_base.hpp +++ b/sdk/storage/test/test_base.hpp @@ -29,6 +29,11 @@ namespace Azure { namespace Storage { namespace Test { void RandomBuffer(char* buffer, std::size_t length); - std::vector ReadBodyStream(Azure::Core::Http::BodyStream* stream); + std::vector ReadBodyStream(std::unique_ptr& stream); + + inline std::vector ReadBodyStream(std::unique_ptr&& stream) + { + return ReadBodyStream(stream); + } }}} // namespace Azure::Storage::Test