Skip to content

Commit c88011c

Browse files
anmolanmol1234anmolanmol1234
committed
HADOOP-18146: ABFS: Added changes for expect hundred continue header (apache#4039)
 This change lets the client react pre-emptively to server load without getting to 503 and the exponential backoff which follows. This stops performance suffering so much as capacity limits are approached for an account. Contributed by Anmol Asranii
1 parent dd9ef9e commit c88011c

27 files changed

+927
-89
lines changed

‎hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,11 @@
4848
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]utils[\\/]Base64.java"/>
4949
<suppress checks="ParameterNumber|VisibilityModifier"
5050
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]ITestSmallWriteOptimization.java"/>
51+
<suppress checks="VisibilityModifier"
52+
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]services[\\/]ITestAbfsRestOperation.java"/>
5153
<!-- allow tests to use _ for ordering. -->
5254
<suppress checks="MethodName"
5355
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]commit[\\/]ITestAbfsTerasort.java"/>
56+
<suppress checks="ParameterNumber"
57+
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]services[\\/]TestAbfsOutputStream.java"/>
5458
</suppressions>

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,11 @@ public class AbfsConfiguration{
117117
DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ)
118118
private boolean optimizeFooterRead;
119119

120+
@BooleanConfigurationValidatorAnnotation(
121+
ConfigurationKey = FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED,
122+
DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED)
123+
private boolean isExpectHeaderEnabled;
124+
120125
@BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED,
121126
DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED)
122127
private boolean accountThrottlingEnabled;
@@ -706,6 +711,10 @@ public String getAppendBlobDirs() {
706711
return this.azureAppendBlobDirs;
707712
}
708713

714+
public boolean isExpectHeaderEnabled() {
715+
return this.isExpectHeaderEnabled;
716+
}
717+
709718
public boolean accountThrottlingEnabled() {
710719
return accountThrottlingEnabled;
711720
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -693,6 +693,7 @@ private AbfsOutputStreamContext populateAbfsOutputStreamContext(
693693
}
694694
return new AbfsOutputStreamContext(abfsConfiguration.getSasTokenRenewPeriodForStreamsInSeconds())
695695
.withWriteBufferSize(bufferSize)
696+
.enableExpectHeader(abfsConfiguration.isExpectHeaderEnabled())
696697
.enableFlush(abfsConfiguration.isFlushEnabled())
697698
.enableSmallWriteOptimization(abfsConfiguration.isSmallWriteOptimizationEnabled())
698699
.disableOutputStreamFlush(abfsConfiguration.isOutputStreamFlushDisabled())

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,11 @@ public final class AbfsHttpConstants {
6464
public static final String HTTP_METHOD_PATCH = "PATCH";
6565
public static final String HTTP_METHOD_POST = "POST";
6666
public static final String HTTP_METHOD_PUT = "PUT";
67+
/**
68+
* All status codes less than http 100 signify error
69+
* and should qualify for retry.
70+
*/
71+
public static final int HTTP_CONTINUE = 100;
6772

6873
// Abfs generic constants
6974
public static final String SINGLE_WHITE_SPACE = " ";
@@ -103,6 +108,9 @@ public final class AbfsHttpConstants {
103108
public static final String DEFAULT_SCOPE = "default:";
104109
public static final String PERMISSION_FORMAT = "%04d";
105110
public static final String SUPER_USER = "$superuser";
111+
// The HTTP 100 Continue informational status response code indicates that everything so far
112+
// is OK and that the client should continue with the request or ignore it if it is already finished.
113+
public static final String HUNDRED_CONTINUE = "100-continue";
106114

107115
public static final char CHAR_FORWARD_SLASH = '/';
108116
public static final char CHAR_EXCLAMATION_POINT = '!';

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ public final class ConfigurationKeys {
3535
* path to determine HNS status.
3636
*/
3737
public static final String FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "fs.azure.account.hns.enabled";
38+
/**
39+
* Enable or disable expect hundred continue header.
40+
* Value: {@value}.
41+
*/
42+
public static final String FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED = "fs.azure.account.expect.header.enabled";
3843
public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key";
3944
public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)";
4045
public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode";

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
public final class FileSystemConfigurations {
3333

3434
public static final String DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "";
35-
35+
public static final boolean DEFAULT_FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED = true;
3636
public static final String USER_HOME_DIRECTORY_PREFIX = "/user";
3737

3838
private static final int SIXTY_SECONDS = 60 * 1000;

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ public final class HttpHeaderConfigurations {
7070
public static final String X_MS_LEASE_ID = "x-ms-lease-id";
7171
public static final String X_MS_PROPOSED_LEASE_ID = "x-ms-proposed-lease-id";
7272
public static final String X_MS_LEASE_BREAK_PERIOD = "x-ms-lease-break-period";
73+
public static final String EXPECT = "Expect";
7374

7475
private HttpHeaderConfigurations() {}
7576
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,33 @@
3030
@InterfaceAudience.Public
3131
@InterfaceStability.Evolving
3232
public class InvalidAbfsRestOperationException extends AbfsRestOperationException {
33+
34+
private static final String ERROR_MESSAGE = "InvalidAbfsRestOperationException";
35+
3336
public InvalidAbfsRestOperationException(
3437
final Exception innerException) {
3538
super(
3639
AzureServiceErrorCode.UNKNOWN.getStatusCode(),
3740
AzureServiceErrorCode.UNKNOWN.getErrorCode(),
3841
innerException != null
3942
? innerException.toString()
40-
: "InvalidAbfsRestOperationException",
43+
: ERROR_MESSAGE,
4144
innerException);
4245
}
46+
47+
/**
48+
* Adds the retry count along with the exception.
49+
* @param innerException The inner exception which is originally caught.
50+
* @param retryCount The retry count when the exception was thrown.
51+
*/
52+
public InvalidAbfsRestOperationException(
53+
final Exception innerException, int retryCount) {
54+
super(
55+
AzureServiceErrorCode.UNKNOWN.getStatusCode(),
56+
AzureServiceErrorCode.UNKNOWN.getErrorCode(),
57+
innerException != null
58+
? innerException.toString()
59+
: ERROR_MESSAGE + " RetryCount: " + retryCount,
60+
innerException);
61+
}
4362
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,19 +34,22 @@ public enum Mode {
3434
private final Mode mode;
3535
private final boolean isAppendBlob;
3636
private final String leaseId;
37+
private boolean isExpectHeaderEnabled;
3738

3839
public AppendRequestParameters(final long position,
3940
final int offset,
4041
final int length,
4142
final Mode mode,
4243
final boolean isAppendBlob,
43-
final String leaseId) {
44+
final String leaseId,
45+
final boolean isExpectHeaderEnabled) {
4446
this.position = position;
4547
this.offset = offset;
4648
this.length = length;
4749
this.mode = mode;
4850
this.isAppendBlob = isAppendBlob;
4951
this.leaseId = leaseId;
52+
this.isExpectHeaderEnabled = isExpectHeaderEnabled;
5053
}
5154

5255
public long getPosition() {
@@ -72,4 +75,12 @@ public boolean isAppendBlob() {
7275
public String getLeaseId() {
7376
return this.leaseId;
7477
}
78+
79+
public boolean isExpectHeaderEnabled() {
80+
return isExpectHeaderEnabled;
81+
}
82+
83+
public void setExpectHeaderEnabled(boolean expectHeaderEnabled) {
84+
isExpectHeaderEnabled = expectHeaderEnabled;
85+
}
7586
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java‎

Lines changed: 79 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME;
7878
import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*;
7979
import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*;
80+
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
8081
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND;
8182

8283
/**
@@ -656,6 +657,9 @@ public AbfsRestOperation append(final String path, final byte[] buffer,
656657
throws AzureBlobFileSystemException {
657658
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
658659
addCustomerProvidedKeyHeaders(requestHeaders);
660+
if (reqParams.isExpectHeaderEnabled()) {
661+
requestHeaders.add(new AbfsHttpHeader(EXPECT, HUNDRED_CONTINUE));
662+
}
659663
// JDK7 does not support PATCH, so to workaround the issue we will use
660664
// PUT and specify the real method in the X-Http-Method-Override header.
661665
requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE,
@@ -681,36 +685,49 @@ public AbfsRestOperation append(final String path, final byte[] buffer,
681685
abfsUriQueryBuilder, cachedSasToken);
682686

683687
final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
684-
final AbfsRestOperation op = new AbfsRestOperation(
685-
AbfsRestOperationType.Append,
686-
this,
687-
HTTP_METHOD_PUT,
688-
url,
689-
requestHeaders,
690-
buffer,
691-
reqParams.getoffset(),
692-
reqParams.getLength(),
693-
sasTokenForReuse);
688+
final AbfsRestOperation op = getAbfsRestOperationForAppend(AbfsRestOperationType.Append,
689+
HTTP_METHOD_PUT,
690+
url,
691+
requestHeaders,
692+
buffer,
693+
reqParams.getoffset(),
694+
reqParams.getLength(),
695+
sasTokenForReuse);
694696
try {
695697
op.execute(tracingContext);
696698
} catch (AzureBlobFileSystemException e) {
699+
/*
700+
If the http response code indicates a user error we retry
701+
the same append request with expect header being disabled.
702+
When "100-continue" header is enabled but a non Http 100 response comes,
703+
the response message might not get set correctly by the server.
704+
So, this handling is to avoid breaking of backward compatibility
705+
if someone has taken dependency on the exception message,
706+
which is created using the error string present in the response header.
707+
*/
708+
int responseStatusCode = ((AbfsRestOperationException) e).getStatusCode();
709+
if (checkUserError(responseStatusCode) && reqParams.isExpectHeaderEnabled()) {
710+
LOG.debug("User error, retrying without 100 continue enabled for the given path {}", path);
711+
reqParams.setExpectHeaderEnabled(false);
712+
return this.append(path, buffer, reqParams, cachedSasToken,
713+
tracingContext);
714+
}
697715
// If we have no HTTP response, throw the original exception.
698716
if (!op.hasResult()) {
699717
throw e;
700718
}
701719
if (reqParams.isAppendBlob()
702720
&& appendSuccessCheckOp(op, path,
703721
(reqParams.getPosition() + reqParams.getLength()), tracingContext)) {
704-
final AbfsRestOperation successOp = new AbfsRestOperation(
705-
AbfsRestOperationType.Append,
706-
this,
707-
HTTP_METHOD_PUT,
708-
url,
709-
requestHeaders,
710-
buffer,
711-
reqParams.getoffset(),
712-
reqParams.getLength(),
713-
sasTokenForReuse);
722+
final AbfsRestOperation successOp = getAbfsRestOperationForAppend(
723+
AbfsRestOperationType.Append,
724+
HTTP_METHOD_PUT,
725+
url,
726+
requestHeaders,
727+
buffer,
728+
reqParams.getoffset(),
729+
reqParams.getLength(),
730+
sasTokenForReuse);
714731
successOp.hardSetResult(HttpURLConnection.HTTP_OK);
715732
return successOp;
716733
}
@@ -720,6 +737,48 @@ && appendSuccessCheckOp(op, path,
720737
return op;
721738
}
722739

740+
/**
741+
* Returns the rest operation for append.
742+
* @param operationType The AbfsRestOperationType.
743+
* @param httpMethod specifies the httpMethod.
744+
* @param url specifies the url.
745+
* @param requestHeaders This includes the list of request headers.
746+
* @param buffer The buffer to write into.
747+
* @param bufferOffset The buffer offset.
748+
* @param bufferLength The buffer Length.
749+
* @param sasTokenForReuse The sasToken.
750+
* @return AbfsRestOperation op.
751+
*/
752+
@VisibleForTesting
753+
AbfsRestOperation getAbfsRestOperationForAppend(final AbfsRestOperationType operationType,
754+
final String httpMethod,
755+
final URL url,
756+
final List<AbfsHttpHeader> requestHeaders,
757+
final byte[] buffer,
758+
final int bufferOffset,
759+
final int bufferLength,
760+
final String sasTokenForReuse) {
761+
return new AbfsRestOperation(
762+
operationType,
763+
this,
764+
httpMethod,
765+
url,
766+
requestHeaders,
767+
buffer,
768+
bufferOffset,
769+
bufferLength, sasTokenForReuse);
770+
}
771+
772+
/**
773+
* Returns true if the status code lies in the range of user error.
774+
* @param responseStatusCode http response status code.
775+
* @return True or False.
776+
*/
777+
private boolean checkUserError(int responseStatusCode) {
778+
return (responseStatusCode >= HttpURLConnection.HTTP_BAD_REQUEST
779+
&& responseStatusCode < HttpURLConnection.HTTP_INTERNAL_ERROR);
780+
}
781+
723782
// For AppendBlob its possible that the append succeeded in the backend but the request failed.
724783
// However a retry would fail with an InvalidQueryParameterValue
725784
// (as the current offset would be unacceptable).

0 commit comments

Comments
 (0)