Skip to content

Commit b0d0b60

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into HADOOP-19691 
2 parents 78ed6e9 + dc7b9e2 commit b0d0b60

File tree

12 files changed

+97
-76
lines changed

12 files changed

+97
-76
lines changed

‎LICENSE-binary‎

Lines changed: 33 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -302,36 +302,36 @@ io.grpc:grpc-stub:1.69.0
302302
io.grpc:grpc-stub:1.70.0
303303
io.grpc:grpc-util:1.70.0
304304
io.grpc:grpc-xds:1.70.0
305-
io.netty:netty-all:4.1.118.Final
306-
io.netty:netty-buffer:4.1.118.Final
307-
io.netty:netty-codec:4.1.118.Final
308-
io.netty:netty-codec-dns:4.1.118.Final
309-
io.netty:netty-codec-haproxy:4.1.118.Final
310-
io.netty:netty-codec-http:4.1.118.Final
311-
io.netty:netty-codec-http2:4.1.118.Final
312-
io.netty:netty-codec-memcache:4.1.118.Final
313-
io.netty:netty-codec-mqtt:4.1.118.Final
314-
io.netty:netty-codec-redis:4.1.118.Final
315-
io.netty:netty-codec-smtp:4.1.118.Final
316-
io.netty:netty-codec-socks:4.1.118.Final
317-
io.netty:netty-codec-stomp:4.1.118.Final
318-
io.netty:netty-codec-xml:4.1.118.Final
319-
io.netty:netty-common:4.1.118.Final
320-
io.netty:netty-handler:4.1.118.Final
321-
io.netty:netty-handler-proxy:4.1.118.Final
322-
io.netty:netty-resolver:4.1.118.Final
323-
io.netty:netty-resolver-dns:4.1.118.Final
324-
io.netty:netty-transport:4.1.118.Final
325-
io.netty:netty-transport-rxtx:4.1.118.Final
326-
io.netty:netty-transport-sctp:4.1.118.Final
327-
io.netty:netty-transport-udt:4.1.118.Final
328-
io.netty:netty-transport-classes-epoll:4.1.118.Final
329-
io.netty:netty-transport-native-unix-common:4.1.118.Final
330-
io.netty:netty-transport-classes-kqueue:4.1.118.Final
331-
io.netty:netty-resolver-dns-classes-macos:4.1.118.Final
332-
io.netty:netty-transport-native-epoll:4.1.118.Final
333-
io.netty:netty-transport-native-kqueue:4.1.118.Final
334-
io.netty:netty-resolver-dns-native-macos:4.1.118.Final
305+
io.netty:netty-all:4.1.127.Final
306+
io.netty:netty-buffer:4.1.127.Final
307+
io.netty:netty-codec:4.1.127.Final
308+
io.netty:netty-codec-dns:4.1.127.Final
309+
io.netty:netty-codec-haproxy:4.1.127.Final
310+
io.netty:netty-codec-http:4.1.127.Final
311+
io.netty:netty-codec-http2:4.1.127.Final
312+
io.netty:netty-codec-memcache:4.1.127.Final
313+
io.netty:netty-codec-mqtt:4.1.127.Final
314+
io.netty:netty-codec-redis:4.1.127.Final
315+
io.netty:netty-codec-smtp:4.1.127.Final
316+
io.netty:netty-codec-socks:4.1.127.Final
317+
io.netty:netty-codec-stomp:4.1.127.Final
318+
io.netty:netty-codec-xml:4.1.127.Final
319+
io.netty:netty-common:4.1.127.Final
320+
io.netty:netty-handler:4.1.127.Final
321+
io.netty:netty-handler-proxy:4.1.127.Final
322+
io.netty:netty-resolver:4.1.127.Final
323+
io.netty:netty-resolver-dns:4.1.127.Final
324+
io.netty:netty-transport:4.1.127.Final
325+
io.netty:netty-transport-rxtx:4.1.127.Final
326+
io.netty:netty-transport-sctp:4.1.127.Final
327+
io.netty:netty-transport-udt:4.1.127.Final
328+
io.netty:netty-transport-classes-epoll:4.1.127.Final
329+
io.netty:netty-transport-native-unix-common:4.1.127.Final
330+
io.netty:netty-transport-classes-kqueue:4.1.127.Final
331+
io.netty:netty-resolver-dns-classes-macos:4.1.127.Final
332+
io.netty:netty-transport-native-epoll:4.1.127.Final
333+
io.netty:netty-transport-native-kqueue:4.1.127.Final
334+
io.netty:netty-resolver-dns-native-macos:4.1.127.Final
335335
io.opencensus:opencensus-api:0.12.3
336336
io.opencensus:opencensus-api:0.31.1
337337
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
@@ -358,9 +358,9 @@ org.apache.commons:commons-compress:1.26.1
358358
org.apache.commons:commons-configuration2:2.10.1
359359
org.apache.commons:commons-csv:1.9.0
360360
org.apache.commons:commons-digester:1.8.1
361-
org.apache.commons:commons-lang3:3.17.0
361+
org.apache.commons:commons-lang3:3.18.0
362362
org.apache.commons:commons-math3:3.6.1
363-
org.apache.commons:commons-text:1.10.0
363+
org.apache.commons:commons-text:1.14.0
364364
org.apache.commons:commons-validator:1.6
365365
org.apache.curator:curator-client:5.2.0
366366
org.apache.curator:curator-framework:5.2.0
@@ -420,6 +420,7 @@ org.yaml:snakeyaml:2.0
420420
org.wildfly.openssl:wildfly-openssl:2.1.4.Final
421421
ro.isdc.wro4j:wro4j-maven-plugin:1.8.0
422422
software.amazon.awssdk:bundle:2.29.52
423+
software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3:1.3.0
423424
net.jodah:failsafe:2.4.4
424425

425426
--------------------------------------------------------------------------------

‎hadoop-common-project/hadoop-common/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@
337337
</dependency>
338338
<dependency>
339339
<groupId>com.squareup.okhttp3</groupId>
340-
<artifactId>mockwebserver</artifactId>
340+
<artifactId>mockwebserver3-junit5</artifactId>
341341
<scope>test</scope>
342342
</dependency>
343343
<dependency>

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818

1919
package org.apache.hadoop.fs.http;
2020

21-
import okhttp3.mockwebserver.MockResponse;
22-
import okhttp3.mockwebserver.MockWebServer;
23-
import okhttp3.mockwebserver.RecordedRequest;
21+
import mockwebserver3.MockResponse;
22+
import mockwebserver3.MockWebServer;
23+
import mockwebserver3.RecordedRequest;
2424
import org.apache.hadoop.conf.Configuration;
2525
import org.apache.hadoop.fs.FileSystem;
2626
import org.apache.hadoop.fs.Path;
@@ -55,7 +55,8 @@ public void testHttpFileSystem() throws IOException, URISyntaxException,
5555
InterruptedException {
5656
final String data = "foo";
5757
try (MockWebServer server = new MockWebServer()) {
58-
IntStream.rangeClosed(1, 3).forEach(i -> server.enqueue(new MockResponse().setBody(data)));
58+
final MockResponse response = new MockResponse.Builder().body(data).build();
59+
IntStream.rangeClosed(1, 3).forEach(i -> server.enqueue(response));
5960
server.start();
6061
URI uri = URI.create(String.format("http://%s:%d", server.getHostName(),
6162
server.getPort()));
@@ -64,7 +65,7 @@ public void testHttpFileSystem() throws IOException, URISyntaxException,
6465
assertSameData(fs, new Path("/foo"), data);
6566
assertSameData(fs, new Path("foo"), data);
6667
RecordedRequest req = server.takeRequest();
67-
assertEquals("/foo", req.getPath());
68+
assertEquals("/foo", req.getUrl().encodedPath());
6869
}
6970
}
7071

‎hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -200,11 +200,6 @@
200200
<artifactId>bcprov-jdk18on</artifactId>
201201
<scope>test</scope>
202202
</dependency>
203-
<dependency>
204-
<groupId>com.squareup.okhttp3</groupId>
205-
<artifactId>mockwebserver</artifactId>
206-
<scope>test</scope>
207-
</dependency>
208203
<dependency>
209204
<groupId>org.jetbrains.kotlin</groupId>
210205
<artifactId>kotlin-stdlib-jdk8</artifactId>

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.slf4j.Logger;
5050
import org.slf4j.LoggerFactory;
5151

52+
import static java.util.Objects.requireNonNull;
5253
import static org.apache.hadoop.fs.FileUtil.maybeIgnoreMissingDirectory;
5354

5455
/**
@@ -455,10 +456,17 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
455456
if (length != 0) {
456457
BlockLocation[] blkLocations;
457458
if (file instanceof LocatedFileStatus) {
458-
blkLocations = ((LocatedFileStatus) file).getBlockLocations();
459+
blkLocations = requireNonNull(
460+
((LocatedFileStatus) file).getBlockLocations(),
461+
() -> String.format("No block locations in LocatedFileStatus %s; length=%d",
462+
file, length));
463+
459464
} else {
460465
FileSystem fs = path.getFileSystem(job.getConfiguration());
461-
blkLocations = fs.getFileBlockLocations(file, 0, length);
466+
blkLocations = requireNonNull(fs.getFileBlockLocations(file, 0, length), () ->
467+
String.format("No block locations returned from getFileBlockLocations(%s, 0, %d)"
468+
+ "; status=%s",
469+
path, length, file));
462470
}
463471
if (isSplitable(job, path)) {
464472
long blockSize = file.getBlockSize();

‎hadoop-project/pom.xml‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,11 @@
127127
<commons-compress.version>1.26.1</commons-compress.version>
128128
<commons-csv.version>1.9.0</commons-csv.version>
129129
<commons-io.version>2.16.1</commons-io.version>
130-
<commons-lang3.version>3.17.0</commons-lang3.version>
130+
<commons-lang3.version>3.18.0</commons-lang3.version>
131131
<commons-logging.version>1.3.0</commons-logging.version>
132132
<commons-math3.version>3.6.1</commons-math3.version>
133133
<commons-net.version>3.9.0</commons-net.version>
134-
<commons-text.version>1.10.0</commons-text.version>
134+
<commons-text.version>1.14.0</commons-text.version>
135135

136136
<kerby.version>2.0.3</kerby.version>
137137
<ehcache.version>3.8.2</ehcache.version>
@@ -140,13 +140,13 @@
140140
<hikari.version>4.0.3</hikari.version>
141141
<derby.version>10.14.2.0</derby.version>
142142
<mssql.version>6.2.1.jre7</mssql.version>
143-
<okhttp3.version>4.11.0</okhttp3.version>
143+
<okhttp3.version>5.1.0</okhttp3.version>
144144
<kotlin-stdlib.version>1.6.20</kotlin-stdlib.version>
145145
<jdom2.version>2.0.6.1</jdom2.version>
146146
<jna.version>5.2.0</jna.version>
147147
<gson.version>2.9.0</gson.version>
148148
<metrics.version>3.2.4</metrics.version>
149-
<netty4.version>4.1.118.Final</netty4.version>
149+
<netty4.version>4.1.127.Final</netty4.version>
150150
<snappy-java.version>1.1.10.4</snappy-java.version>
151151
<lz4-java.version>1.7.1</lz4-java.version>
152152
<byte-buddy.version>1.17.6</byte-buddy.version>
@@ -219,7 +219,7 @@
219219
<aws-java-sdk.version>1.12.720</aws-java-sdk.version>
220220
<aws-java-sdk-v2.version>2.29.52</aws-java-sdk-v2.version>
221221
<amazon-s3-encryption-client-java.version>3.1.1</amazon-s3-encryption-client-java.version>
222-
<amazon-s3-analyticsaccelerator-s3.version>1.2.1</amazon-s3-analyticsaccelerator-s3.version>
222+
<amazon-s3-analyticsaccelerator-s3.version>1.3.0</amazon-s3-analyticsaccelerator-s3.version>
223223
<aws.eventstream.version>1.0.1</aws.eventstream.version>
224224
<hsqldb.version>2.7.1</hsqldb.version>
225225
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
@@ -256,7 +256,7 @@
256256
<dependencies>
257257
<dependency>
258258
<groupId>com.squareup.okhttp3</groupId>
259-
<artifactId>mockwebserver</artifactId>
259+
<artifactId>mockwebserver3-junit5</artifactId>
260260
<version>${okhttp3.version}</version>
261261
<scope>test</scope>
262262
<exclusions>
@@ -1392,6 +1392,12 @@
13921392
<groupId>com.googlecode.json-simple</groupId>
13931393
<artifactId>json-simple</artifactId>
13941394
<version>1.1.1</version>
1395+
<exclusions>
1396+
<exclusion>
1397+
<groupId>junit</groupId>
1398+
<artifactId>junit</artifactId>
1399+
</exclusion>
1400+
</exclusions>
13951401
</dependency>
13961402

13971403
<dependency>

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ALocatedFileStatus.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,15 +95,15 @@ public S3AFileStatus toS3AFileStatus() {
9595

9696
@Override
9797
public String toString() {
98-
final StringBuilder sb = new StringBuilder(
99-
super.toString());
98+
final StringBuilder sb = new StringBuilder("S3ALocatedFileStatus{");
99+
sb.append(super.toString());
100100
sb.append("[eTag='").
101101
append(eTag != null ? eTag : "")
102102
.append('\'');
103103
sb.append(", versionId='")
104104
.append(versionId != null ? versionId: "")
105105
.append('\'');
106-
sb.append(']');
106+
sb.append('}');
107107
return sb.toString();
108108
}
109109
}

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -993,10 +993,10 @@ public InputStreamType streamType() {
993993
private class FactoryCallbacks implements StreamFactoryCallbacks {
994994

995995
@Override
996-
public S3AsyncClient getOrCreateAsyncClient(final boolean requireCRT) throws IOException {
996+
public S3Client getOrCreateSyncClient() throws IOException {
997997
// Needs support of the CRT before the requireCRT can be used
998998
LOG.debug("Stream factory requested async client");
999-
return clientManager().getOrCreateAsyncClient();
999+
return clientManager().getOrCreateS3Client();
10001000
}
10011001

10021002
@Override

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStreamFactory.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@
2323

2424
import org.slf4j.Logger;
2525
import org.slf4j.LoggerFactory;
26-
import software.amazon.s3.analyticsaccelerator.S3SdkObjectClient;
26+
2727
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamConfiguration;
2828
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamFactory;
29+
import software.amazon.s3.analyticsaccelerator.S3SyncSdkObjectClient;
2930
import software.amazon.s3.analyticsaccelerator.common.ConnectorConfiguration;
3031

3132
import org.apache.hadoop.conf.Configuration;
@@ -118,7 +119,7 @@ private S3SeekableInputStreamFactory getOrCreateS3SeekableInputStreamFactory()
118119

119120
private CallableRaisingIOE<S3SeekableInputStreamFactory> createS3SeekableInputStreamFactory() {
120121
return () -> new S3SeekableInputStreamFactory(
121-
new S3SdkObjectClient(callbacks().getOrCreateAsyncClient(requireCrt)),
122+
new S3SyncSdkObjectClient(callbacks().getOrCreateSyncClient()),
122123
seekableInputStreamConfiguration);
123124
}
124125

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/ObjectInputStreamFactory.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
import java.io.IOException;
2222

23-
import software.amazon.awssdk.services.s3.S3AsyncClient;
23+
import software.amazon.awssdk.services.s3.S3Client;
2424

2525
import org.apache.hadoop.fs.s3a.Statistic;
2626
import org.apache.hadoop.fs.StreamCapabilities;
@@ -80,12 +80,11 @@ ObjectInputStream readObject(ObjectReadParameters parameters)
8080
interface StreamFactoryCallbacks {
8181

8282
/**
83-
* Get the Async S3Client, raising a failure to create as an IOException.
84-
* @param requireCRT is the CRT required.
83+
* Get the Sync S3Client, raising a failure to create as an IOException.
8584
* @return the Async S3 client
8685
* @throws IOException failure to create the client.
8786
*/
88-
S3AsyncClient getOrCreateAsyncClient(boolean requireCRT) throws IOException;
87+
S3Client getOrCreateSyncClient() throws IOException;
8988

9089
void incrementFactoryStatistic(Statistic statistic);
9190
}

0 commit comments

Comments
 (0)