Skip to content

Commit 71a1fe9

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into HADOOP-19136 
2 parents d3c2185 + a1ae35e commit 71a1fe9

File tree

17 files changed

+467
-40
lines changed

17 files changed

+467
-40
lines changed

‎LICENSE-binary‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,6 @@ org.apache.commons:commons-validator:1.6
310310
org.apache.curator:curator-client:5.2.0
311311
org.apache.curator:curator-framework:5.2.0
312312
org.apache.curator:curator-recipes:5.2.0
313-
org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
314313
org.apache.hbase:hbase-annotations:1.7.1
315314
org.apache.hbase:hbase-client:1.7.1
316315
org.apache.hbase:hbase-common:1.7.1
@@ -507,6 +506,7 @@ com.sun.jersey:jersey-server:1.19.4
507506
com.sun.jersey:jersey-servlet:1.19.4
508507
com.sun.xml.bind:jaxb-impl:2.2.3-1
509508
javax.annotation:javax.annotation-api:1.3.2
509+
javax.cache:cache-api:1.1.1
510510
javax.servlet:javax.servlet-api:3.1.0
511511
javax.servlet.jsp:jsp-api:2.1
512512
javax.websocket:javax.websocket-api:1.0

‎hadoop-project/pom.xml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,8 @@
132132
<commons-text.version>1.10.0</commons-text.version>
133133

134134
<kerby.version>2.0.3</kerby.version>
135-
<jcache.version>1.0-alpha-1</jcache.version>
136135
<ehcache.version>3.8.2</ehcache.version>
136+
<cache.api.version>1.1.1</cache.api.version>
137137
<hikari.version>4.0.3</hikari.version>
138138
<derby.version>10.14.2.0</derby.version>
139139
<mssql.version>6.2.1.jre7</mssql.version>
@@ -1943,11 +1943,6 @@
19431943
<artifactId>kerb-core</artifactId>
19441944
<version>${kerby.version}</version>
19451945
</dependency>
1946-
<dependency>
1947-
<groupId>org.apache.geronimo.specs</groupId>
1948-
<artifactId>geronimo-jcache_1.0_spec</artifactId>
1949-
<version>${jcache.version}</version>
1950-
</dependency>
19511946
<dependency>
19521947
<groupId>org.ehcache</groupId>
19531948
<artifactId>ehcache</artifactId>
@@ -2078,6 +2073,11 @@
20782073
<artifactId>log4j-web</artifactId>
20792074
<version>${log4j2.version}</version>
20802075
</dependency>
2076+
<dependency>
2077+
<groupId>javax.cache</groupId>
2078+
<artifactId>cache-api</artifactId>
2079+
<version>${cache.api.version}</version>
2080+
</dependency>
20812081
</dependencies>
20822082
</dependencyManagement>
20832083

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1423,12 +1423,12 @@ private Constants() {
14231423
/**
14241424
* Default minimum seek in bytes during vectored reads : {@value}.
14251425
*/
1426-
public static final int DEFAULT_AWS_S3_VECTOR_READS_MIN_SEEK_SIZE = 4896; // 4K
1426+
public static final int DEFAULT_AWS_S3_VECTOR_READS_MIN_SEEK_SIZE = 4096; // 4K
14271427

14281428
/**
14291429
* Default maximum read size in bytes during vectored reads : {@value}.
14301430
*/
1431-
public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M
1431+
public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1048576; //1M
14321432

14331433
/**
14341434
* Maximum number of range reads a single input stream can have

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,10 @@ public class AbfsConfiguration{
363363
FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, DefaultValue = DEFAULT_ENABLE_ABFS_CHECKSUM_VALIDATION)
364364
private boolean isChecksumValidationEnabled;
365365

366+
@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
367+
FS_AZURE_ENABLE_PAGINATED_DELETE, DefaultValue = DEFAULT_ENABLE_PAGINATED_DELETE)
368+
private boolean isPaginatedDeleteEnabled;
369+
366370
private String clientProvidedEncryptionKey;
367371
private String clientProvidedEncryptionKeySHA;
368372

@@ -1240,8 +1244,8 @@ public boolean getRenameResilience() {
12401244
return renameResilience;
12411245
}
12421246

1243-
void setRenameResilience(boolean actualResilience) {
1244-
renameResilience = actualResilience;
1247+
public boolean isPaginatedDeleteEnabled() {
1248+
return isPaginatedDeleteEnabled;
12451249
}
12461250

12471251
public boolean getIsChecksumValidationEnabled() {

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,8 +1077,8 @@ public void delete(final Path path, final boolean recursive,
10771077

10781078
do {
10791079
try (AbfsPerfInfo perfInfo = startTracking("delete", "deletePath")) {
1080-
AbfsRestOperation op = client
1081-
.deletePath(relativePath, recursive, continuation, tracingContext);
1080+
AbfsRestOperation op = client.deletePath(relativePath, recursive,
1081+
continuation, tracingContext, getIsNamespaceEnabled(tracingContext));
10821082
perfInfo.registerResult(op.getResult());
10831083
continuation = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION);
10841084
perfInfo.registerSuccess(true);

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,37 @@ public final class AbfsHttpConstants {
121121
public static final char CHAR_EQUALS = '=';
122122
public static final char CHAR_STAR = '*';
123123
public static final char CHAR_PLUS = '+';
124-
public static final String DECEMBER_2019_API_VERSION = "2019-12-12";
125-
public static final String APRIL_2021_API_VERSION = "2021-04-10";
124+
125+
/**
126+
* Specifies the version of the REST protocol used for processing the request.
127+
* Versions should be added in enum list in ascending chronological order.
128+
* Latest one should be added last in the list.
129+
* When upgrading the version for whole driver, update the getCurrentVersion;
130+
*/
131+
public enum ApiVersion {
132+
133+
DEC_12_2019("2019-12-12"),
134+
APR_10_2021("2021-04-10"),
135+
AUG_03_2023("2023-08-03");
136+
137+
private final String xMsApiVersion;
138+
139+
ApiVersion(String xMsApiVersion) {
140+
this.xMsApiVersion = xMsApiVersion;
141+
}
142+
143+
@Override
144+
public String toString() {
145+
return xMsApiVersion;
146+
}
147+
148+
public static ApiVersion getCurrentVersion() {
149+
return DEC_12_2019;
150+
}
151+
}
152+
153+
@Deprecated
154+
public static final String DECEMBER_2019_API_VERSION = ApiVersion.DEC_12_2019.toString();
126155

127156
/**
128157
* Value that differentiates categories of the http_status.

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,11 @@ public final class ConfigurationKeys {
275275
/** Add extra resilience to rename failures, at the expense of performance. */
276276
public static final String FS_AZURE_ABFS_RENAME_RESILIENCE = "fs.azure.enable.rename.resilience";
277277

278+
/**
279+
* Specify whether paginated behavior is to be expected or not in delete path. {@value}
280+
*/
281+
public static final String FS_AZURE_ENABLE_PAGINATED_DELETE = "fs.azure.enable.paginated.delete";
282+
278283
/** Add extra layer of verification of the integrity of the request content during transport: {@value}. */
279284
public static final String FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION = "fs.azure.enable.checksum.validation";
280285

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ public final class FileSystemConfigurations {
133133
public static final int STREAM_ID_LEN = 12;
134134
public static final boolean DEFAULT_ENABLE_ABFS_LIST_ITERATOR = true;
135135
public static final boolean DEFAULT_ENABLE_ABFS_RENAME_RESILIENCE = true;
136+
public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false;
136137
public static final boolean DEFAULT_ENABLE_ABFS_CHECKSUM_VALIDATION = false;
137138

138139
/**

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public final class HttpQueryParams {
4040
public static final String QUERY_PARAM_CLOSE = "close";
4141
public static final String QUERY_PARAM_UPN = "upn";
4242
public static final String QUERY_PARAM_BLOBTYPE = "blobtype";
43+
public static final String QUERY_PARAM_PAGINATED = "paginated";
4344

4445
//query params for SAS
4546
public static final String QUERY_PARAM_SAOID = "saoid";

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public class AbfsClient implements Closeable {
100100

101101
private final URL baseUrl;
102102
private final SharedKeyCredentials sharedKeyCredentials;
103-
private String xMsVersion = DECEMBER_2019_API_VERSION;
103+
private ApiVersion xMsVersion = ApiVersion.getCurrentVersion();
104104
private final ExponentialRetryPolicy exponentialRetryPolicy;
105105
private final StaticRetryPolicy staticRetryPolicy;
106106
private final String filesystem;
@@ -122,7 +122,6 @@ public class AbfsClient implements Closeable {
122122
private final ListeningScheduledExecutorService executorService;
123123
private Boolean isNamespaceEnabled;
124124

125-
126125
private boolean renameResilience;
127126

128127
/**
@@ -149,7 +148,7 @@ private AbfsClient(final URL baseUrl,
149148

150149
if (encryptionContextProvider != null) {
151150
this.encryptionContextProvider = encryptionContextProvider;
152-
xMsVersion = APRIL_2021_API_VERSION; // will be default once server change deployed
151+
xMsVersion = ApiVersion.APR_10_2021; // will be default once server change deployed
153152
encryptionType = EncryptionType.ENCRYPTION_CONTEXT;
154153
} else if (abfsConfiguration.getEncodedClientProvidedEncryptionKey() != null) {
155154
clientProvidedEncryptionKey =
@@ -259,13 +258,27 @@ AbfsThrottlingIntercept getIntercept() {
259258
return intercept;
260259
}
261260

262-
List<AbfsHttpHeader> createDefaultHeaders() {
261+
/**
262+
* Create request headers for Rest Operation using the current API version.
263+
* @return default request headers
264+
*/
265+
@VisibleForTesting
266+
protected List<AbfsHttpHeader> createDefaultHeaders() {
267+
return createDefaultHeaders(this.xMsVersion);
268+
}
269+
270+
/**
271+
* Create request headers for Rest Operation using the specified API version.
272+
* @param xMsVersion
273+
* @return default request headers
274+
*/
275+
private List<AbfsHttpHeader> createDefaultHeaders(ApiVersion xMsVersion) {
263276
final List<AbfsHttpHeader> requestHeaders = new ArrayList<AbfsHttpHeader>();
264-
requestHeaders.add(new AbfsHttpHeader(X_MS_VERSION, xMsVersion));
277+
requestHeaders.add(new AbfsHttpHeader(X_MS_VERSION, xMsVersion.toString()));
265278
requestHeaders.add(new AbfsHttpHeader(ACCEPT, APPLICATION_JSON
266-
+ COMMA + SINGLE_WHITE_SPACE + APPLICATION_OCTET_STREAM));
279+
+ COMMA + SINGLE_WHITE_SPACE + APPLICATION_OCTET_STREAM));
267280
requestHeaders.add(new AbfsHttpHeader(ACCEPT_CHARSET,
268-
UTF_8));
281+
UTF_8));
269282
requestHeaders.add(new AbfsHttpHeader(CONTENT_TYPE, EMPTY_STRING));
270283
requestHeaders.add(new AbfsHttpHeader(USER_AGENT, userAgent));
271284
return requestHeaders;
@@ -1117,12 +1130,29 @@ public AbfsRestOperation read(final String path,
11171130
return op;
11181131
}
11191132

1120-
public AbfsRestOperation deletePath(final String path, final boolean recursive, final String continuation,
1121-
TracingContext tracingContext)
1133+
public AbfsRestOperation deletePath(final String path, final boolean recursive,
1134+
final String continuation,
1135+
TracingContext tracingContext,
1136+
final boolean isNamespaceEnabled)
11221137
throws AzureBlobFileSystemException {
1123-
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
1124-
1138+
/*
1139+
* If Pagination is enabled and current API version is old,
1140+
* use the minimum required version for pagination.
1141+
* If Pagination is enabled and current API version is later than minimum required
1142+
* version for pagination, use current version only as azure service is backward compatible.
1143+
* If pagination is disabled, use the current API version only.
1144+
*/
1145+
final List<AbfsHttpHeader> requestHeaders = (isPaginatedDelete(recursive,
1146+
isNamespaceEnabled) && xMsVersion.compareTo(ApiVersion.AUG_03_2023) < 0)
1147+
? createDefaultHeaders(ApiVersion.AUG_03_2023)
1148+
: createDefaultHeaders();
11251149
final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
1150+
1151+
if (isPaginatedDelete(recursive, isNamespaceEnabled)) {
1152+
// Add paginated query parameter
1153+
abfsUriQueryBuilder.addQuery(QUERY_PARAM_PAGINATED, TRUE);
1154+
}
1155+
11261156
abfsUriQueryBuilder.addQuery(QUERY_PARAM_RECURSIVE, String.valueOf(recursive));
11271157
abfsUriQueryBuilder.addQuery(QUERY_PARAM_CONTINUATION, continuation);
11281158
String operation = recursive ? SASTokenProvider.DELETE_RECURSIVE_OPERATION : SASTokenProvider.DELETE_OPERATION;
@@ -1465,6 +1495,14 @@ private synchronized Boolean getIsNamespaceEnabled(TracingContext tracingContext
14651495
return isNamespaceEnabled;
14661496
}
14671497

1498+
protected Boolean getIsPaginatedDeleteEnabled() {
1499+
return abfsConfiguration.isPaginatedDeleteEnabled();
1500+
}
1501+
1502+
private Boolean isPaginatedDelete(boolean isRecursiveDelete, boolean isNamespaceEnabled) {
1503+
return getIsPaginatedDeleteEnabled() && isNamespaceEnabled && isRecursiveDelete;
1504+
}
1505+
14681506
public AuthType getAuthType() {
14691507
return authType;
14701508
}
@@ -1659,7 +1697,7 @@ protected AbfsCounters getAbfsCounters() {
16591697
return abfsCounters;
16601698
}
16611699

1662-
public String getxMsVersion() {
1700+
public ApiVersion getxMsVersion() {
16631701
return xMsVersion;
16641702
}
16651703

0 commit comments

Comments
 (0)