Skip to content

Commit

Permalink
Upload incorrect length (#13144)
Browse files Browse the repository at this point in the history
* Fixed a couple bugs

* CI

* CI

* CI
  • Loading branch information
rickle-msft authored Jul 15, 2020
1 parent b144116 commit 679c08a
Show file tree
Hide file tree
Showing 8 changed files with 401 additions and 10 deletions.
3 changes: 2 additions & 1 deletion sdk/storage/azure-storage-blob/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Release History

## 12.8.0-beta.2 (Unreleased)

- Fixed a bug that, when the data length parameter did not match the actual length of the data in BlobClient.upload, caused a zero length blob to be uploaded rather than throwing an exception.
- Fixed a bug that ignored the customer's specified block size when determining buffer sizes in BlobClient.upload

## 12.8.0-beta.1 (2020-07-07)
- Added support for the 2019-12-12 service version.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,9 @@ public Mono<Response<BlockBlobItem>> uploadWithResponse(BlobParallelUploadOption
.setTier(options.getTier()).setRequestConditions(options.getRequestConditions()));

Flux<ByteBuffer> data = options.getDataFlux() == null ? Utility.convertStreamToByteBuffer(
options.getDataStream(), options.getLength(), BLOB_DEFAULT_UPLOAD_BLOCK_SIZE)
options.getDataStream(), options.getLength(),
// We can only buffer up to max int due to restrictions in ByteBuffer.
(int) Math.min(Integer.MAX_VALUE, validatedParallelTransferOptions.getBlockSizeLong()))
: options.getDataFlux();
return UploadUtils.uploadFullOrChunked(data, ModelHelper.wrapBlobOptions(validatedParallelTransferOptions),
uploadInChunksFunction, uploadFullBlobMethod);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,9 @@ class BlobAPITest extends APISpec {
}

/* Tests an issue found where buffered upload would not deep copy buffers while determining what upload path to take. */

@Unroll
def "Upload input stream single upload" () {
def "Upload input stream single upload"() {
setup:
def randomData = getRandomByteArray(20 * Constants.KB)
def input = new ByteArrayInputStream(randomData)
Expand All @@ -100,15 +101,15 @@ class BlobAPITest extends APISpec {
stream.toByteArray() == randomData

where:
size || _
1 * Constants.KB || _ /* Less than copyToOutputStream buffer size, Less than maxSingleUploadSize */
8 * Constants.KB || _ /* Equal to copyToOutputStream buffer size, Less than maxSingleUploadSize */
20 * Constants.KB || _ /* Greater than copyToOutputStream buffer size, Less than maxSingleUploadSize */
size || _
1 * Constants.KB || _ /* Less than copyToOutputStream buffer size, Less than maxSingleUploadSize */
8 * Constants.KB || _ /* Equal to copyToOutputStream buffer size, Less than maxSingleUploadSize */
20 * Constants.KB || _ /* Greater than copyToOutputStream buffer size, Less than maxSingleUploadSize */
}

/* TODO (gapra): Add more tests to test large data sizes. */

@Requires( { liveMode() } )
@Requires({ liveMode() })
def "Upload input stream large data"() {
setup:
def randomData = getRandomByteArray(20 * Constants.MB)
Expand All @@ -124,6 +125,22 @@ class BlobAPITest extends APISpec {
notThrown(BlobStorageException)
}

@Unroll
def "Upload incorrect size"() {
when:
bc.upload(defaultInputStream.get(), dataSize, true)

then:
thrown(IllegalStateException)

where:
dataSize | threshold
defaultDataSize + 1 | null
defaultDataSize - 1 | null
defaultDataSize + 1 | 1 // Test the chunked case as well
defaultDataSize - 1 | 1
}

@Unroll
def "Upload numBlocks"() {
setup:
Expand Down Expand Up @@ -156,7 +173,8 @@ class BlobAPITest extends APISpec {
.getValue().getETag() != null
}

@Requires({ liveMode() }) // Reading from recordings will not allow for the timing of the test to work correctly.
@Requires({ liveMode() })
// Reading from recordings will not allow for the timing of the test to work correctly.
def "Upload timeout"() {
setup:
def size = 1024
Expand Down Expand Up @@ -973,7 +991,8 @@ class BlobAPITest extends APISpec {
This test requires two accounts that are configured in a very specific way. It is not feasible to setup that
relationship programmatically, so we have recorded a successful interaction and only test recordings.
*/
@Requires( {playbackMode()})

@Requires({ playbackMode() })
def "Get properties ORS"() {
setup:
def sourceBlob = primaryBlobServiceClient.getBlobContainerClient("test1")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"networkCallRecords" : [ {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "6bf35f33-b5b3-409c-8dc2-5f130fb97710"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"ETag" : "0x8D82819FF1A5384",
"Last-Modified" : "Tue, 14 Jul 2020 17:18:59 GMT",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "201",
"x-ms-request-id" : "d46d6efa-901e-0054-2b02-5abe5e000000",
"Date" : "Tue, 14 Jul 2020 17:18:59 GMT",
"x-ms-client-request-id" : "6bf35f33-b5b3-409c-8dc2-5f130fb97710"
},
"Exception" : null
}, {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717/javablobuploadincorrectsize192960ae99c1dbb885442",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "0addb259-10d1-4120-888b-ca7ae000dbb7",
"Content-Type" : "application/octet-stream"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"x-ms-content-crc64" : "6RYQPwaVsyQ=",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:00 GMT",
"retry-after" : "0",
"StatusCode" : "201",
"x-ms-request-server-encrypted" : "true",
"Date" : "Tue, 14 Jul 2020 17:18:59 GMT",
"Content-MD5" : "wh+Wm18D0z1D4E+PE252gg==",
"ETag" : "0x8D82819FFBE88FD",
"Content-Length" : "0",
"x-ms-request-id" : "b2e2badf-b01e-0061-7402-5a100b000000",
"x-ms-client-request-id" : "0addb259-10d1-4120-888b-ca7ae000dbb7"
},
"Exception" : null
}, {
"Method" : "GET",
"Uri" : "https://REDACTED.blob.core.windows.net?prefix=jtcuploadincorrectsize&comp=list",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "3062524d-62b2-4ecf-8aa6-b0466c650321"
},
"Response" : {
"Transfer-Encoding" : "chunked",
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"StatusCode" : "200",
"x-ms-request-id" : "3bcebc16-c01e-006e-3e02-5afdfd000000",
"Body" : "<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://xclientdev3.blob.core.windows.net/\"><Prefix>jtcuploadincorrectsize</Prefix><Containers><Container><Name>jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717</Name><Properties><Last-Modified>Tue, 14 Jul 2020 17:18:59 GMT</Last-Modified><Etag>\"0x8D82819FF1A5384\"</Etag><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><DefaultEncryptionScope>$account-encryption-key</DefaultEncryptionScope><DenyEncryptionScopeOverride>false</DenyEncryptionScopeOverride><HasImmutabilityPolicy>false</HasImmutabilityPolicy><HasLegalHold>false</HasLegalHold></Properties></Container></Containers><NextMarker /></EnumerationResults>",
"Date" : "Tue, 14 Jul 2020 17:19:00 GMT",
"x-ms-client-request-id" : "3062524d-62b2-4ecf-8aa6-b0466c650321",
"Content-Type" : "application/xml"
},
"Exception" : null
}, {
"Method" : "DELETE",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "bf4ee0d9-d744-4434-86cd-d42dae8ef27a"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "202",
"x-ms-request-id" : "8f04ebd6-801e-0040-0402-5a7d3a000000",
"Date" : "Tue, 14 Jul 2020 17:19:01 GMT",
"x-ms-client-request-id" : "bf4ee0d9-d744-4434-86cd-d42dae8ef27a"
},
"Exception" : null
} ],
"variables" : [ "jtcuploadincorrectsize0blobapitestuploadincorrectsize61a372717", "javablobuploadincorrectsize192960ae99c1dbb885442" ]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"networkCallRecords" : [ {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "3357dd18-4743-45cc-8c81-678638cd39d0"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"ETag" : "0x8D8281A00C3DE5B",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:02 GMT",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "201",
"x-ms-request-id" : "4e84195f-e01e-009c-5b02-5a2f69000000",
"Date" : "Tue, 14 Jul 2020 17:19:01 GMT",
"x-ms-client-request-id" : "3357dd18-4743-45cc-8c81-678638cd39d0"
},
"Exception" : null
}, {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537/javablobuploadincorrectsize105800ad666ed1358e41c",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "b65064d9-44e9-4637-8afc-f1c27f9f1a99",
"Content-Type" : "application/octet-stream"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"x-ms-content-crc64" : "6RYQPwaVsyQ=",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:02 GMT",
"retry-after" : "0",
"StatusCode" : "201",
"x-ms-request-server-encrypted" : "true",
"Date" : "Tue, 14 Jul 2020 17:19:01 GMT",
"Content-MD5" : "wh+Wm18D0z1D4E+PE252gg==",
"ETag" : "0x8D8281A00F2D3DC",
"Content-Length" : "0",
"x-ms-request-id" : "b9dba07a-101e-000a-1102-5a4d5d000000",
"x-ms-client-request-id" : "b65064d9-44e9-4637-8afc-f1c27f9f1a99"
},
"Exception" : null
}, {
"Method" : "GET",
"Uri" : "https://REDACTED.blob.core.windows.net?prefix=jtcuploadincorrectsize&comp=list",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "039695a3-cdd6-4f4f-bf61-0607b7441c14"
},
"Response" : {
"Transfer-Encoding" : "chunked",
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"StatusCode" : "200",
"x-ms-request-id" : "919d11bb-201e-002b-5202-5a206c000000",
"Body" : "<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://xclientdev3.blob.core.windows.net/\"><Prefix>jtcuploadincorrectsize</Prefix><Containers><Container><Name>jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537</Name><Properties><Last-Modified>Tue, 14 Jul 2020 17:19:02 GMT</Last-Modified><Etag>\"0x8D8281A00C3DE5B\"</Etag><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><DefaultEncryptionScope>$account-encryption-key</DefaultEncryptionScope><DenyEncryptionScopeOverride>false</DenyEncryptionScopeOverride><HasImmutabilityPolicy>false</HasImmutabilityPolicy><HasLegalHold>false</HasLegalHold></Properties></Container></Containers><NextMarker /></EnumerationResults>",
"Date" : "Tue, 14 Jul 2020 17:19:02 GMT",
"x-ms-client-request-id" : "039695a3-cdd6-4f4f-bf61-0607b7441c14",
"Content-Type" : "application/xml"
},
"Exception" : null
}, {
"Method" : "DELETE",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "44c31a46-1731-4413-a8a8-08a3dee3d7c6"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "202",
"x-ms-request-id" : "77df9095-901e-0010-4502-5a6232000000",
"Date" : "Tue, 14 Jul 2020 17:19:02 GMT",
"x-ms-client-request-id" : "44c31a46-1731-4413-a8a8-08a3dee3d7c6"
},
"Exception" : null
} ],
"variables" : [ "jtcuploadincorrectsize0blobapitestuploadincorrectsizec47250537", "javablobuploadincorrectsize105800ad666ed1358e41c" ]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"networkCallRecords" : [ {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "d37e8d0e-2ed0-4aff-817f-5aecbe772444"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"ETag" : "0x8D8281A01899BB6",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:03 GMT",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "201",
"x-ms-request-id" : "167610af-a01e-0031-6502-5a0f03000000",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"x-ms-client-request-id" : "d37e8d0e-2ed0-4aff-817f-5aecbe772444"
},
"Exception" : null
}, {
"Method" : "PUT",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d/javablobuploadincorrectsize1958500e125872c20b445",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "70a6acea-8a74-46ec-a8bf-dd19d4162bbe",
"Content-Type" : "application/octet-stream"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"x-ms-content-crc64" : "6RYQPwaVsyQ=",
"Last-Modified" : "Tue, 14 Jul 2020 17:19:03 GMT",
"retry-after" : "0",
"StatusCode" : "201",
"x-ms-request-server-encrypted" : "true",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"Content-MD5" : "wh+Wm18D0z1D4E+PE252gg==",
"ETag" : "0x8D8281A01B9A32A",
"Content-Length" : "0",
"x-ms-request-id" : "d2c931c3-d01e-00f2-7c02-5a8640000000",
"x-ms-client-request-id" : "70a6acea-8a74-46ec-a8bf-dd19d4162bbe"
},
"Exception" : null
}, {
"Method" : "GET",
"Uri" : "https://REDACTED.blob.core.windows.net?prefix=jtcuploadincorrectsize&comp=list",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "bdd56435-50ac-4727-9134-747e0bed473e"
},
"Response" : {
"Transfer-Encoding" : "chunked",
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"StatusCode" : "200",
"x-ms-request-id" : "8193fad7-d01e-003e-6d02-5ae2f5000000",
"Body" : "<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://xclientdev3.blob.core.windows.net/\"><Prefix>jtcuploadincorrectsize</Prefix><Containers><Container><Name>jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d</Name><Properties><Last-Modified>Tue, 14 Jul 2020 17:19:03 GMT</Last-Modified><Etag>\"0x8D8281A01899BB6\"</Etag><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><DefaultEncryptionScope>$account-encryption-key</DefaultEncryptionScope><DenyEncryptionScopeOverride>false</DenyEncryptionScopeOverride><HasImmutabilityPolicy>false</HasImmutabilityPolicy><HasLegalHold>false</HasLegalHold></Properties></Container></Containers><NextMarker /></EnumerationResults>",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"x-ms-client-request-id" : "bdd56435-50ac-4727-9134-747e0bed473e",
"Content-Type" : "application/xml"
},
"Exception" : null
}, {
"Method" : "DELETE",
"Uri" : "https://REDACTED.blob.core.windows.net/jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d?restype=container",
"Headers" : {
"x-ms-version" : "2019-12-12",
"User-Agent" : "azsdk-java-azure-storage-blob/12.8.0-beta.2 (11.0.7; Windows 10; 10.0)",
"x-ms-client-request-id" : "cc682a78-2410-4fa2-b4af-8489aeb9b645"
},
"Response" : {
"x-ms-version" : "2019-12-12",
"Server" : "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0",
"retry-after" : "0",
"Content-Length" : "0",
"StatusCode" : "202",
"x-ms-request-id" : "a98ba26c-301e-013b-0902-5a50df000000",
"Date" : "Tue, 14 Jul 2020 17:19:03 GMT",
"x-ms-client-request-id" : "cc682a78-2410-4fa2-b4af-8489aeb9b645"
},
"Exception" : null
} ],
"variables" : [ "jtcuploadincorrectsize0blobapitestuploadincorrectsizea0271001d", "javablobuploadincorrectsize1958500e125872c20b445" ]
}
Loading

0 comments on commit 679c08a

Please sign in to comment.