Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upload incorrect length #13144

Merged
merged 4 commits into from
Jul 15, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion sdk/storage/azure-storage-blob/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Release History

## 12.8.0-beta.2 (Unreleased)

- Fixed a bug that, when the data length parameter did not match the actual length of the data in BlobClient.upload, caused a zero length blob to be uploaded rather than throwing an exception.
- Fixed a bug that ignored the customer's specified block size when determining buffer sizes in BlobClient.upload

## 12.8.0-beta.1 (2020-07-07)
- Added support for the 2019-12-12 service version.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,9 @@ public Mono<Response<BlockBlobItem>> uploadWithResponse(BlobParallelUploadOption
.setTier(options.getTier()).setRequestConditions(options.getRequestConditions()));

Flux<ByteBuffer> data = options.getDataFlux() == null ? Utility.convertStreamToByteBuffer(
options.getDataStream(), options.getLength(), BLOB_DEFAULT_UPLOAD_BLOCK_SIZE)
options.getDataStream(), options.getLength(),
// We can only buffer up to max int due to restrictions in ByteBuffer.
(int) Math.min(Integer.MAX_VALUE, validatedParallelTransferOptions.getBlockSizeLong()))
: options.getDataFlux();
return UploadUtils.uploadFullOrChunked(data, ModelHelper.wrapBlobOptions(validatedParallelTransferOptions),
uploadInChunksFunction, uploadFullBlobMethod);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,9 @@ class BlobAPITest extends APISpec {
}

/* Tests an issue found where buffered upload would not deep copy buffers while determining what upload path to take. */

@Unroll
def "Upload input stream single upload" () {
def "Upload input stream single upload"() {
setup:
def randomData = getRandomByteArray(20 * Constants.KB)
def input = new ByteArrayInputStream(randomData)
Expand All @@ -100,15 +101,15 @@ class BlobAPITest extends APISpec {
stream.toByteArray() == randomData

where:
size || _
1 * Constants.KB || _ /* Less than copyToOutputStream buffer size, Less than maxSingleUploadSize */
8 * Constants.KB || _ /* Equal to copyToOutputStream buffer size, Less than maxSingleUploadSize */
20 * Constants.KB || _ /* Greater than copyToOutputStream buffer size, Less than maxSingleUploadSize */
size || _
1 * Constants.KB || _ /* Less than copyToOutputStream buffer size, Less than maxSingleUploadSize */
8 * Constants.KB || _ /* Equal to copyToOutputStream buffer size, Less than maxSingleUploadSize */
20 * Constants.KB || _ /* Greater than copyToOutputStream buffer size, Less than maxSingleUploadSize */
}

/* TODO (gapra): Add more tests to test large data sizes. */

@Requires( { liveMode() } )
@Requires({ liveMode() })
def "Upload input stream large data"() {
setup:
def randomData = getRandomByteArray(20 * Constants.MB)
Expand All @@ -124,6 +125,22 @@ class BlobAPITest extends APISpec {
notThrown(BlobStorageException)
}

@Unroll
def "Upload incorrect size"() {
when:
bc.upload(defaultInputStream.get(), dataSize, true)

then:
thrown(IllegalStateException)

where:
dataSize | threshold
defaultDataSize + 1 | null
defaultDataSize - 1 | null
defaultDataSize + 1 | 1 // Test the chunked case as well
defaultDataSize - 1 | 1
}

@Unroll
def "Upload numBlocks"() {
setup:
Expand Down Expand Up @@ -156,7 +173,8 @@ class BlobAPITest extends APISpec {
.getValue().getETag() != null
}

@Requires({ liveMode() }) // Reading from recordings will not allow for the timing of the test to work correctly.
@Requires({ liveMode() })
// Reading from recordings will not allow for the timing of the test to work correctly.
def "Upload timeout"() {
setup:
def size = 1024
Expand Down Expand Up @@ -973,7 +991,8 @@ class BlobAPITest extends APISpec {
This test requires two accounts that are configured in a very specific way. It is not feasible to setup that
relationship programmatically, so we have recorded a successful interaction and only test recordings.
*/
@Requires( {playbackMode()})

@Requires({ playbackMode() })
def "Get properties ORS"() {
setup:
def sourceBlob = primaryBlobServiceClient.getBlobContainerClient("test1")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"networkCallRecords" : [ {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "6bf35f33-b5b3-409c-8dc2-5f130fb97710"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"ETag" : "0x8D82819FF1A5384",
"Last-Modified" : "Tue, 14 Jul 2020 17:18:59 GMT",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "201",
"x-ms-request-id" : "d46d6efa-901e-0054-2b02-5abe5e000000",
"Date" : "Tue, 14 Jul 2020 17:18:59 GMT",
"x-ms-client-request-id" : "6bf35f33-b5b3-409c-8dc2-5f130fb97710"
},
"Exception" : null
}, {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717/javablobuploadincorrectsize192960ae99c1dbb885442",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "0addb259-10d1-4120-888b-ca7ae000dbb7",
"Content-Type" : "application/octet-stream"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"x-ms-content-crc64" : "6RYQPwaVsyQ=",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:00 GMT",
"retry-after" : "0",
"StatusCode" : "201",
"x-ms-request-server-encrypted" : "true",
"Date" : "Tue, 14 Jul 2020 17:18:59 GMT",
"Content-MD5" : "wh+Wm18D0z1D4E+PE252gg==",
"ETag" : "0x8D82819FFBE88FD",
"Content-Length" : "0",
"x-ms-request-id" : "b2e2badf-b01e-0061-7402-5a100b000000",
"x-ms-client-request-id" : "0addb259-10d1-4120-888b-ca7ae000dbb7"
},
"Exception" : null
}, {
"Method" : "GET",
"Uri" : "https://REDACTED.blob.core.windows.net?prefix=jtcuploadincorrectsize&comp=list",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "3062524d-62b2-4ecf-8aa6-b0466c650321"
},
"Response" : {
"Transfer-Encoding" : "chunked",
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"StatusCode" : "200",
"x-ms-request-id" : "3bcebc16-c01e-006e-3e02-5afdfd000000",
"Body" : "<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://xclientdev3.blob.core.windows.net/\"><Prefix>jtcuploadincorrectsize</Prefix><Containers><Container><Name>jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717</Name><Properties><Last-Modified>Tue, 14 Jul 2020 17:18:59 GMT</Last-Modified><Etag>\"0x8D82819FF1A5384\"</Etag><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><DefaultEncryptionScope>$account-encryption-key</DefaultEncryptionScope><DenyEncryptionScopeOverride>false</DenyEncryptionScopeOverride><HasImmutabilityPolicy>false</HasImmutabilityPolicy><HasLegalHold>false</HasLegalHold></Properties></Container></Containers><NextMarker /></EnumerationResults>",
"Date" : "Tue, 14 Jul 2020 17:19:00 GMT",
"x-ms-client-request-id" : "3062524d-62b2-4ecf-8aa6-b0466c650321",
"Content-Type" : "application/xml"
},
"Exception" : null
}, {
"Method" : "DELETE",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "bf4ee0d9-d744-4434-86cd-d42dae8ef27a"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "202",
"x-ms-request-id" : "8f04ebd6-801e-0040-0402-5a7d3a000000",
"Date" : "Tue, 14 Jul 2020 17:19:01 GMT",
"x-ms-client-request-id" : "bf4ee0d9-d744-4434-86cd-d42dae8ef27a"
},
"Exception" : null
} ],
"variables" : [ "jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717", "javablobuploadincorrectsize192960ae99c1dbb885442" ]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"networkCallRecords" : [ {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "3357dd18-4743-45cc-8c81-678638cd39d0"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"ETag" : "0x8D8281A00C3DE5B",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:02 GMT",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "201",
"x-ms-request-id" : "4e84195f-e01e-009c-5b02-5a2f69000000",
"Date" : "Tue, 14 Jul 2020 17:19:01 GMT",
"x-ms-client-request-id" : "3357dd18-4743-45cc-8c81-678638cd39d0"
},
"Exception" : null
}, {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537/javablobuploadincorrectsize105800ad666ed1358e41c",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "b65064d9-44e9-4637-8afc-f1c27f9f1a99",
"Content-Type" : "application/octet-stream"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"x-ms-content-crc64" : "6RYQPwaVsyQ=",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:02 GMT",
"retry-after" : "0",
"StatusCode" : "201",
"x-ms-request-server-encrypted" : "true",
"Date" : "Tue, 14 Jul 2020 17:19:01 GMT",
"Content-MD5" : "wh+Wm18D0z1D4E+PE252gg==",
"ETag" : "0x8D8281A00F2D3DC",
"Content-Length" : "0",
"x-ms-request-id" : "b9dba07a-101e-000a-1102-5a4d5d000000",
"x-ms-client-request-id" : "b65064d9-44e9-4637-8afc-f1c27f9f1a99"
},
"Exception" : null
}, {
"Method" : "GET",
"Uri" : "https://REDACTED.blob.core.windows.net?prefix=jtcuploadincorrectsize&comp=list",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "039695a3-cdd6-4f4f-bf61-0607b7441c14"
},
"Response" : {
"Transfer-Encoding" : "chunked",
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"StatusCode" : "200",
"x-ms-request-id" : "919d11bb-201e-002b-5202-5a206c000000",
"Body" : "<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://xclientdev3.blob.core.windows.net/\"><Prefix>jtcuploadincorrectsize</Prefix><Containers><Container><Name>jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537</Name><Properties><Last-Modified>Tue, 14 Jul 2020 17:19:02 GMT</Last-Modified><Etag>\"0x8D8281A00C3DE5B\"</Etag><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><DefaultEncryptionScope>$account-encryption-key</DefaultEncryptionScope><DenyEncryptionScopeOverride>false</DenyEncryptionScopeOverride><HasImmutabilityPolicy>false</HasImmutabilityPolicy><HasLegalHold>false</HasLegalHold></Properties></Container></Containers><NextMarker /></EnumerationResults>",
"Date" : "Tue, 14 Jul 2020 17:19:02 GMT",
"x-ms-client-request-id" : "039695a3-cdd6-4f4f-bf61-0607b7441c14",
"Content-Type" : "application/xml"
},
"Exception" : null
}, {
"Method" : "DELETE",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "44c31a46-1731-4413-a8a8-08a3dee3d7c6"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "202",
"x-ms-request-id" : "77df9095-901e-0010-4502-5a6232000000",
"Date" : "Tue, 14 Jul 2020 17:19:02 GMT",
"x-ms-client-request-id" : "44c31a46-1731-4413-a8a8-08a3dee3d7c6"
},
"Exception" : null
} ],
"variables" : [ "jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537", "javablobuploadincorrectsize105800ad666ed1358e41c" ]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"networkCallRecords" : [ {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "d37e8d0e-2ed0-4aff-817f-5aecbe772444"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"ETag" : "0x8D8281A01899BB6",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:03 GMT",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "201",
"x-ms-request-id" : "167610af-a01e-0031-6502-5a0f03000000",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"x-ms-client-request-id" : "d37e8d0e-2ed0-4aff-817f-5aecbe772444"
},
"Exception" : null
}, {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d/javablobuploadincorrectsize1958500e125872c20b445",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "70a6acea-8a74-46ec-a8bf-dd19d4162bbe",
"Content-Type" : "application/octet-stream"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"x-ms-content-crc64" : "6RYQPwaVsyQ=",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:03 GMT",
"retry-after" : "0",
"StatusCode" : "201",
"x-ms-request-server-encrypted" : "true",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"Content-MD5" : "wh+Wm18D0z1D4E+PE252gg==",
"ETag" : "0x8D8281A01B9A32A",
"Content-Length" : "0",
"x-ms-request-id" : "d2c931c3-d01e-00f2-7c02-5a8640000000",
"x-ms-client-request-id" : "70a6acea-8a74-46ec-a8bf-dd19d4162bbe"
},
"Exception" : null
}, {
"Method" : "GET",
"Uri" : "https://REDACTED.blob.core.windows.net?prefix=jtcuploadincorrectsize&comp=list",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "bdd56435-50ac-4727-9134-747e0bed473e"
},
"Response" : {
"Transfer-Encoding" : "chunked",
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"StatusCode" : "200",
"x-ms-request-id" : "8193fad7-d01e-003e-6d02-5ae2f5000000",
"Body" : "<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://xclientdev3.blob.core.windows.net/\"><Prefix>jtcuploadincorrectsize</Prefix><Containers><Container><Name>jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d</Name><Properties><Last-Modified>Tue, 14 Jul 2020 17:19:03 GMT</Last-Modified><Etag>\"0x8D8281A01899BB6\"</Etag><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><DefaultEncryptionScope>$account-encryption-key</DefaultEncryptionScope><DenyEncryptionScopeOverride>false</DenyEncryptionScopeOverride><HasImmutabilityPolicy>false</HasImmutabilityPolicy><HasLegalHold>false</HasLegalHold></Properties></Container></Containers><NextMarker /></EnumerationResults>",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"x-ms-client-request-id" : "bdd56435-50ac-4727-9134-747e0bed473e",
"Content-Type" : "application/xml"
},
"Exception" : null
}, {
"Method" : "DELETE",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "cc682a78-2410-4fa2-b4af-8489aeb9b645"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "202",
"x-ms-request-id" : "a98ba26c-301e-013b-0902-5a50df000000",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"x-ms-client-request-id" : "cc682a78-2410-4fa2-b4af-8489aeb9b645"
},
"Exception" : null
} ],
"variables" : [ "jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d", "javablobuploadincorrectsize1958500e125872c20b445" ]
}
Loading