Skip to content

HADOOP-16570. S3A committers encounter scale issues #1442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
steveloughran wants to merge 12 commits into from
Closed

HADOOP-16570. S3A committers encounter scale issues #1442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
11bd3bc
HADOOP-16570. S3A committers leak threads on job/task commit.
steveloughran Sep 13, 2019
b3cfaa2
HADOOP-16570 expand thread leakage checks
steveloughran Sep 18, 2019
25ff1ce
HADOOP-16570 Incremental loading and committing of .pendingset files.
steveloughran Sep 20, 2019
b2ab98e
HADOOP-15670 ongoing work on revert logic
steveloughran Sep 24, 2019
64dc205
HADOOP-16570. Complete (mostly) the move two parallel Reading of .pen…
steveloughran Sep 24, 2019
7a23a81
HADOOP-16570 add a test to simulate scale commits.
steveloughran Sep 26, 2019
2079a34
HADOOP-16570 tuning the new test
steveloughran Sep 26, 2019
90825f5
HADOOP-16570 log duration of the load/commit/abort operations to iden…
steveloughran Sep 26, 2019
27fa7bc
HADOOP-16570 Staging committers preload pendingset files for prefligh…
steveloughran Sep 30, 2019
b5195d5
HADOOP-16570 feedback, especially on Tasks use
steveloughran Oct 1, 2019
ac41d9e
HADOOP-16570: final review feedback from Ewan
steveloughran Oct 4, 2019
3734adc
HADOOP-16570 final tunings from test run review
steveloughran Oct 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -837,4 +837,10 @@ private Constants() {
public static final String AWS_SERVICE_IDENTIFIER_S3 = "S3";
public static final String AWS_SERVICE_IDENTIFIER_DDB = "DDB";
public static final String AWS_SERVICE_IDENTIFIER_STS = "STS";

/**
* How long to wait for the thread pool to terminate when cleaning up.
* Value: {@value} seconds.
*/
public static final int THREAD_POOL_SHUTDOWN_DELAY_SECONDS = 30;
}
11 changes: 9 additions & 2 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.SemaphoredDelegatingExecutor;
import org.apache.hadoop.util.concurrent.HadoopExecutors;

import static org.apache.hadoop.fs.impl.AbstractFSBuilderImpl.rejectUnknownMandatoryKeys;
import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
Expand Down Expand Up @@ -3062,6 +3063,12 @@ public void close() throws IOException {
transfers.shutdownNow(true);
transfers = null;
}
HadoopExecutors.shutdown(boundedThreadPool, LOG,
THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
boundedThreadPool = null;
HadoopExecutors.shutdown(unboundedThreadPool, LOG,
THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
unboundedThreadPool = null;
S3AUtils.closeAll(LOG, metadataStore, instrumentation);
metadataStore = null;
instrumentation = null;
Expand Down Expand Up @@ -4064,7 +4071,7 @@ public List<MultipartUpload> listMultipartUploads(String prefix)
*/
@Retries.OnceRaw
void abortMultipartUpload(String destKey, String uploadId) {
LOG.info("Aborting multipart upload {} to {}", uploadId, destKey);
LOG.debug("Aborting multipart upload {} to {}", uploadId, destKey);
getAmazonS3Client().abortMultipartUpload(
new AbortMultipartUploadRequest(getBucket(),
destKey,
Expand All @@ -4084,7 +4091,7 @@ void abortMultipartUpload(MultipartUpload upload) {
uploadId = upload.getUploadId();
if (LOG.isInfoEnabled()) {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LOG.info("Aborting multipart upload {} to {} initiated by {} on {}",
LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}",
uploadId, destKey, upload.getInitiator(),
df.format(upload.getInitiated()));
}
Expand Down
3 changes: 3 additions & 0 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,11 +611,14 @@ public void getMetrics(MetricsCollector collector, boolean all) {

public void close() {
synchronized (metricsSystemLock) {
// it is critical to close each quantile, as they start a scheduled
// task in a shared thread pool.
putLatencyQuantile.stop();
throttleRateQuantile.stop();
metricsSystem.unregisterSource(metricsSourceName);
int activeSources = --metricsSourceActiveCounter;
if (activeSources == 0) {
LOG.debug("Shutting down metrics publisher");
metricsSystem.publishMetricsNow();
metricsSystem.shutdown();
metricsSystem = null;
Expand Down
Loading