-
Notifications
You must be signed in to change notification settings - Fork 549
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix multipart upload pause #1536
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
/** | ||
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"). | ||
* You may not use this file except in compliance with the License. | ||
* A copy of the License is located at | ||
* | ||
* http://aws.amazon.com/apache2.0 | ||
* | ||
* or in the "license" file accompanying this file. This file is distributed | ||
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
* express or implied. See the License for the specific language governing | ||
* permissions and limitations under the License. | ||
*/ | ||
|
||
package com.amazonaws.mobileconnectors.s3.transferutility; | ||
|
||
import android.content.Context; | ||
import android.support.test.InstrumentationRegistry; | ||
|
||
import com.amazonaws.services.s3.S3IntegrationTestBase; | ||
|
||
import org.junit.AfterClass; | ||
import org.junit.Before; | ||
import org.junit.BeforeClass; | ||
import org.junit.Test; | ||
|
||
import java.io.File; | ||
import java.util.Date; | ||
import java.util.concurrent.CountDownLatch; | ||
import java.util.concurrent.TimeUnit; | ||
|
||
public final class PauseTransferIntegrationTest extends S3IntegrationTestBase { | ||
|
||
private static final String bucketName = "amazon-transfer-util-integ-test-" + new Date().getTime(); | ||
private static TransferUtility util; | ||
private static Context context = InstrumentationRegistry.getContext(); | ||
|
||
private File file; | ||
private CountDownLatch started; | ||
private CountDownLatch paused; | ||
private CountDownLatch resumed; | ||
private CountDownLatch completed; | ||
|
||
/** | ||
* Creates and initializes all the test resources needed for these tests. | ||
*/ | ||
@BeforeClass | ||
public static void setUpBeforeClass() throws Exception { | ||
setUp(); | ||
TransferNetworkLossHandler.getInstance(context); | ||
util = TransferUtility.builder() | ||
.context(context) | ||
.s3Client(s3) | ||
.build(); | ||
|
||
try { | ||
s3.createBucket(bucketName); | ||
waitForBucketCreation(bucketName); | ||
} catch (final Exception e) { | ||
System.out.println("Error in creating the bucket. " | ||
+ "Please manually create the bucket " + bucketName); | ||
} | ||
} | ||
|
||
@AfterClass | ||
public static void tearDown() { | ||
try { | ||
deleteBucketAndAllContents(bucketName); | ||
} catch (final Exception e) { | ||
System.out.println("Error in deleting the bucket. " | ||
+ "Please manually delete the bucket " + bucketName); | ||
e.printStackTrace(); | ||
} | ||
} | ||
|
||
@Before | ||
public void setUpLatches() { | ||
started = new CountDownLatch(1); | ||
paused = new CountDownLatch(1); | ||
resumed = new CountDownLatch(1); | ||
completed = new CountDownLatch(1); | ||
} | ||
|
||
@Test | ||
public void testSinglePartUploadPause() throws Exception { | ||
// Small (1KB) file upload | ||
file = getRandomTempFile("small", 1000L); | ||
testUploadPause(); | ||
} | ||
|
||
@Test | ||
public void testMultiPartUploadPause() throws Exception { | ||
// Large (10MB) file upload | ||
file = getRandomSparseFile("large", 10L * 1024 * 1024); | ||
testUploadPause(); | ||
} | ||
|
||
private void testUploadPause() throws Exception { | ||
// start transfer and wait for progress | ||
TransferObserver observer = util.upload(bucketName, file.getName(), file); | ||
observer.setTransferListener(new TestListener()); | ||
started.await(100, TimeUnit.MILLISECONDS); | ||
|
||
// pause and wait | ||
util.pause(observer.getId()); | ||
paused.await(100, TimeUnit.MILLISECONDS); | ||
Thread.sleep(1000); // throws if progress is made | ||
|
||
// resume if pause was properly executed | ||
util.resume(observer.getId()); | ||
resumed.await(100, TimeUnit.MILLISECONDS); | ||
|
||
// cancel early to avoid having to wait for completion | ||
util.cancel(observer.getId()); | ||
completed.await(100, TimeUnit.MILLISECONDS); | ||
} | ||
|
||
private final class TestListener implements TransferListener { | ||
@Override | ||
public void onStateChanged(int id, TransferState state) { | ||
switch (state) { | ||
case CANCELED: | ||
case COMPLETED: | ||
completed.countDown(); | ||
break; | ||
case PAUSED: | ||
paused.countDown(); | ||
break; | ||
case IN_PROGRESS: | ||
if (paused.getCount() == 0) { | ||
// Post-pause | ||
resumed.countDown(); | ||
} else { | ||
// Pre-pause | ||
started.countDown(); | ||
} | ||
break; | ||
case FAILED: | ||
throw new RuntimeException("Failed transfer."); | ||
} | ||
} | ||
|
||
@Override | ||
public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) { | ||
if (paused.getCount() == 0 && resumed.getCount() > 0) { | ||
throw new RuntimeException("Progress made even while paused."); | ||
} | ||
} | ||
|
||
@Override | ||
public void onError(int id, Exception ex) { | ||
throw new RuntimeException(ex); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -114,16 +114,16 @@ public Boolean call() { | |
return true; | ||
} catch (final Exception e) { | ||
// No need to update the progress listener. | ||
if (TransferState.CANCELED.equals(download.state)) { | ||
LOGGER.info("Transfer is " + download.state); | ||
if (TransferState.PENDING_CANCEL.equals(download.state)) { | ||
updater.updateState(download.id, TransferState.CANCELED); | ||
LOGGER.info("Transfer is " + TransferState.CANCELED); | ||
return false; | ||
} | ||
|
||
// Reset the progress when the transfer is paused. | ||
if (TransferState.PAUSED.equals(download.state)) { | ||
LOGGER.info("Transfer is " + download.state); | ||
ProgressEvent resetEvent = new ProgressEvent(0); | ||
resetEvent.setEventCode(ProgressEvent.RESET_EVENT_CODE); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are we not resetting on pause on anymore? If so, do we need to update documentation? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if you look closely, the lines i removed were noops. Im not sure if the original intent was to pass in a |
||
if (TransferState.PENDING_PAUSE.equals(download.state)) { | ||
updater.updateState(download.id, TransferState.PAUSED); | ||
LOGGER.info("Transfer is " + TransferState.PAUSED); | ||
progressListener.progressChanged(new ProgressEvent(0)); | ||
return false; | ||
} | ||
|
@@ -144,8 +144,6 @@ public Boolean call() { | |
*/ | ||
updater.updateState(download.id, TransferState.WAITING_FOR_NETWORK); | ||
LOGGER.debug("Network Connection Interrupted: " + "Moving the TransferState to WAITING_FOR_NETWORK"); | ||
ProgressEvent resetEvent = new ProgressEvent(0); | ||
resetEvent.setEventCode(ProgressEvent.RESET_EVENT_CODE); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't we be resetting the progress upon loss of network and then pass |
||
progressListener.progressChanged(new ProgressEvent(0)); | ||
return false; | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,7 +30,6 @@ | |
import com.amazonaws.services.s3.model.ObjectTagging; | ||
import com.amazonaws.services.s3.model.PartETag; | ||
import com.amazonaws.services.s3.model.PutObjectRequest; | ||
import com.amazonaws.services.s3.model.PutObjectResult; | ||
import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams; | ||
import com.amazonaws.services.s3.model.Tag; | ||
import com.amazonaws.services.s3.model.UploadPartRequest; | ||
|
@@ -193,15 +192,6 @@ private Boolean uploadMultipartAndWaitForCompletion() throws ExecutionException | |
} catch (final Exception e) { | ||
LOGGER.error("Upload resulted in an exception. " + e); | ||
|
||
// If the thread that is executing the transfer is interrupted | ||
// because of a user initiated pause or cancel operation, | ||
// do not throw exception or set the state to FAILED. | ||
if (TransferState.CANCELED.equals(upload.state) || | ||
TransferState.PAUSED.equals(upload.state)) { | ||
LOGGER.info("Transfer is " + upload.state); | ||
return false; | ||
} | ||
|
||
/* | ||
* Future.get() will catch InterruptedException, but it's not a | ||
* failure, it may be caused by a pause operation from applications. | ||
|
@@ -211,6 +201,21 @@ private Boolean uploadMultipartAndWaitForCompletion() throws ExecutionException | |
task.uploadPartTask.cancel(true); | ||
} | ||
|
||
// If the thread that is executing the transfer is interrupted | ||
// because of a user initiated pause or cancel operation, | ||
// do not throw exception or set the state to FAILED. | ||
if (TransferState.PENDING_CANCEL.equals(upload.state)) { | ||
updater.updateState(upload.id, TransferState.CANCELED); | ||
LOGGER.info("Transfer is " + TransferState.CANCELED); | ||
return false; | ||
} | ||
|
||
if (TransferState.PENDING_PAUSE.equals(upload.state)) { | ||
updater.updateState(upload.id, TransferState.PAUSED); | ||
LOGGER.info("Transfer is " + TransferState.PAUSED); | ||
return false; | ||
} | ||
|
||
// interrupted due to network. Set the TransferState to | ||
// WAITING_FOR_NETWORK if the individual parts were waiting for network | ||
for (final UploadPartTaskMetadata task : uploadPartTasks.values()) { | ||
|
@@ -274,22 +279,22 @@ private Boolean uploadSinglePartAndWaitForCompletion() { | |
putObjectRequest.setGeneralProgressListener(progressListener); | ||
|
||
try { | ||
PutObjectResult putObjectResult = s3.putObject(putObjectRequest); | ||
s3.putObject(putObjectRequest); | ||
updater.updateProgress(upload.id, length, length, true); | ||
updater.updateState(upload.id, TransferState.COMPLETED); | ||
return true; | ||
} catch (final Exception e) { | ||
// we dont need to update progress listener | ||
if (TransferState.CANCELED.equals(upload.state)) { | ||
LOGGER.info("Transfer is " + upload.state); | ||
// we dont need to update progress listener | ||
if (TransferState.PENDING_CANCEL.equals(upload.state)) { | ||
updater.updateState(upload.id, TransferState.CANCELED); | ||
LOGGER.info("Transfer is " + TransferState.CANCELED); | ||
return false; | ||
} | ||
|
||
// pause | ||
if (TransferState.PAUSED.equals(upload.state)) { | ||
LOGGER.info("Transfer is " + upload.state); | ||
ProgressEvent resetEvent = new ProgressEvent(0); | ||
resetEvent.setEventCode(ProgressEvent.RESET_EVENT_CODE); | ||
if (TransferState.PENDING_PAUSE.equals(upload.state)) { | ||
updater.updateState(upload.id, TransferState.PAUSED); | ||
LOGGER.info("Transfer is " + TransferState.PAUSED); | ||
progressListener.progressChanged(new ProgressEvent(0)); | ||
return false; | ||
} | ||
|
@@ -310,8 +315,6 @@ private Boolean uploadSinglePartAndWaitForCompletion() { | |
*/ | ||
updater.updateState(upload.id, TransferState.WAITING_FOR_NETWORK); | ||
LOGGER.debug("Network Connection Interrupted: " + "Moving the TransferState to WAITING_FOR_NETWORK"); | ||
ProgressEvent resetEvent = new ProgressEvent(0); | ||
resetEvent.setEventCode(ProgressEvent.RESET_EVENT_CODE); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as above. If |
||
progressListener.progressChanged(new ProgressEvent(0)); | ||
return false; | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we name it like
BUCKET_NAME
to be consistent with convention and rest of the codebase?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yea oops. A mistake that was copy + pasted from my previous mistake. will fix them