Skip to content

Commit 8925c4a

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into HADOOP-19636 
2 parents a789f69 + 0d7efae commit 8925c4a

File tree

84 files changed

+1855
-1318
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+1855
-1318
lines changed

‎hadoop-client-modules/hadoop-client-minicluster/pom.xml‎

Lines changed: 66 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@
124124
<groupId>javax.servlet</groupId>
125125
<artifactId>javax.servlet-api</artifactId>
126126
</exclusion>
127+
<exclusion>
128+
<groupId>jakarta.servlet</groupId>
129+
<artifactId>jakarta.servlet-api</artifactId>
130+
</exclusion>
131+
<exclusion>
132+
<groupId>jakarta.ws.rs</groupId>
133+
<artifactId>jakarta.ws.rs-api</artifactId>
134+
</exclusion>
127135
<exclusion>
128136
<groupId>javax.xml.bind</groupId>
129137
<artifactId>jaxb-api</artifactId>
@@ -224,10 +232,6 @@
224232
<groupId>commons-codec</groupId>
225233
<artifactId>commons-codec</artifactId>
226234
</exclusion>
227-
<exclusion>
228-
<groupId>javax.servlet</groupId>
229-
<artifactId>javax.servlet-api</artifactId>
230-
</exclusion>
231235

232236
<!-- removing dependency jars from yarn-server-common, which are
233237
already included in hadoop-client-runtime and hadoop-client-api
@@ -415,22 +419,42 @@
415419
<optional>true</optional>
416420
</dependency>
417421
<!-- Add back in the transitive dependencies excluded from hadoop-common in client TODO remove once we have a filter for "is in these artifacts" -->
418-
<!-- skip javax.servlet:servlet-api because it's in client -->
422+
<!-- skip javax.servlet:javax.servlet-api because it's in client -->
423+
<!-- skip jakarta.servlet:jakarta.servlet-api because it's in client -->
424+
<!-- skip jakarta.ws.rs-api:jakarta.ws.rs because it's in client -->
419425
<!-- Skip commons-logging:commons-logging-api because it looks like nothing actually included it -->
420426
<!-- Skip jetty-util because it's in client -->
421427
<dependency>
422428
<groupId>org.glassfish.jersey.core</groupId>
423429
<artifactId>jersey-common</artifactId>
430+
<exclusions>
431+
<exclusion>
432+
<groupId>jakarta.ws.rs</groupId>
433+
<artifactId>jakarta.ws.rs-api</artifactId>
434+
</exclusion>
435+
</exclusions>
424436
<optional>true</optional>
425437
</dependency>
426438
<dependency>
427-
<groupId>org.glassfish.jersey.core</groupId>
428-
<artifactId>jersey-client</artifactId>
439+
<groupId>org.glassfish.jersey.core</groupId>
440+
<artifactId>jersey-client</artifactId>
441+
<exclusions>
442+
<exclusion>
443+
<groupId>jakarta.ws.rs</groupId>
444+
<artifactId>jakarta.ws.rs-api</artifactId>
445+
</exclusion>
446+
</exclusions>
429447
<optional>true</optional>
430448
</dependency>
431449
<dependency>
432450
<groupId>org.glassfish.jersey.core</groupId>
433451
<artifactId>jersey-server</artifactId>
452+
<exclusions>
453+
<exclusion>
454+
<groupId>jakarta.ws.rs</groupId>
455+
<artifactId>jakarta.ws.rs-api</artifactId>
456+
</exclusion>
457+
</exclusions>
434458
<optional>true</optional>
435459
</dependency>
436460
<!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
@@ -445,7 +469,9 @@
445469
<optional>true</optional>
446470
</dependency>
447471
<!-- add back in transitive dependencies of hadoop-mapreduce-client-app removed in client -->
448-
<!-- Skipping javax.servlet:servlet-api because it's in client -->
472+
<!-- Skipping javax.servlet:javax.servlet-api because it's in client -->
473+
<!-- Skipping jakarta.servlet:jakarta.servlet-api because it's in client -->
474+
<!-- Skipping jakarta.ws.rs-api:jakarta.ws.rs because it's in client -->
449475
<dependency>
450476
<groupId>org.apache.hadoop</groupId>
451477
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
@@ -455,6 +481,14 @@
455481
<groupId>javax.servlet</groupId>
456482
<artifactId>javax.servlet-api</artifactId>
457483
</exclusion>
484+
<exclusion>
485+
<groupId>jakarta.servlet</groupId>
486+
<artifactId>jakarta.servlet-api</artifactId>
487+
</exclusion>
488+
<exclusion>
489+
<groupId>jakarta.ws.rs</groupId>
490+
<artifactId>jakarta.ws.rs-api</artifactId>
491+
</exclusion>
458492
<exclusion>
459493
<groupId>org.apache.hadoop</groupId>
460494
<artifactId>hadoop-yarn-api</artifactId>
@@ -522,6 +556,14 @@
522556
<groupId>javax.servlet</groupId>
523557
<artifactId>javax.servlet-api</artifactId>
524558
</exclusion>
559+
<exclusion>
560+
<groupId>jakarta.servlet</groupId>
561+
<artifactId>jakarta.servlet-api</artifactId>
562+
</exclusion>
563+
<exclusion>
564+
<groupId>jakarta.ws.rs</groupId>
565+
<artifactId>jakarta.ws.rs-api</artifactId>
566+
</exclusion>
525567
<exclusion>
526568
<groupId>org.apache.hadoop</groupId>
527569
<artifactId>hadoop-yarn-api</artifactId>
@@ -578,11 +620,27 @@
578620
<dependency>
579621
<groupId>org.glassfish.jersey.test-framework</groupId>
580622
<artifactId>jersey-test-framework-core</artifactId>
623+
<exclusions>
624+
<exclusion>
625+
<groupId>jakarta.ws.rs</groupId>
626+
<artifactId>jakarta.ws.rs-api</artifactId>
627+
</exclusion>
628+
<exclusion>
629+
<groupId>jakarta.servlet</groupId>
630+
<artifactId>jakarta.servlet-api</artifactId>
631+
</exclusion>
632+
</exclusions>
581633
<optional>true</optional>
582634
</dependency>
583635
<dependency>
584636
<groupId>org.glassfish.jersey.test-framework.providers</groupId>
585637
<artifactId>jersey-test-framework-provider-jetty</artifactId>
638+
<exclusions>
639+
<exclusion>
640+
<groupId>jakarta.ws.rs</groupId>
641+
<artifactId>jakarta.ws.rs-api</artifactId>
642+
</exclusion>
643+
</exclusions>
586644
<optional>true</optional>
587645
</dependency>
588646
<!-- skipping jersey-server because it's above -->
@@ -655,9 +713,6 @@
655713
<!-- Leave snappy that includes native methods which cannot be relocated. -->
656714
<exclude>org.xerial.snappy:*</exclude>
657715
<exclude>org.glassfish.jersey:*</exclude>
658-
<exclude>org.hamcrest:*</exclude>
659-
<exclude>aopalliance:*</exclude>
660-
<exclude>javassist:*</exclude>
661716
</excludes>
662717
</artifactSet>
663718
<filters>

‎hadoop-client-modules/hadoop-client-runtime/pom.xml‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,14 +176,11 @@
176176
<exclude>org.glassfish.jersey.core:*</exclude>
177177
<exclude>org.glassfish.hk2.external:*</exclude>
178178
<exclude>org.glassfish.jaxb:*</exclude>
179-
<exclude>jakarta.ws.rs:*</exclude>
180179
<exclude>jakarta.annotation:*</exclude>
181180
<exclude>jakarta.validation:*</exclude>
182-
<exclude>jakarta.servlet:*</exclude>
183-
<exclude>javax.annotation:*</exclude>
184181
<exclude>org.hamcrest:*</exclude>
182+
<exclude>org.javassist:*</exclude>
185183
<exclude>aopalliance:*</exclude>
186-
<exclude>javassist:*</exclude>
187184
</excludes>
188185
</artifactSet>
189186
<filters>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,6 @@
116116
<scope>test</scope>
117117
<type>test-jar</type>
118118
</dependency>
119-
<dependency>
120-
<groupId>junit</groupId>
121-
<artifactId>junit</artifactId>
122-
<version>${junit.version}</version>
123-
<scope>test</scope>
124-
</dependency>
125119
<dependency>
126120
<groupId>org.mockito</groupId>
127121
<artifactId>mockito-all</artifactId>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/src/test/java/org/apache/hadoop/fs/obs/TestOBSContractCreate.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
import org.apache.hadoop.conf.Configuration;
2222
import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
2323
import org.apache.hadoop.fs.contract.AbstractFSContract;
24-
import org.junit.Assume;
24+
25+
import org.junit.jupiter.api.Disabled;
2526

2627
/**
2728
* Create test cases on obs file system.
@@ -33,13 +34,13 @@ protected AbstractFSContract createContract(final Configuration conf) {
3334
return new OBSContract(conf);
3435
}
3536

37+
@Disabled
3638
@Override
3739
public void testCreatedFileIsImmediatelyVisible() {
38-
Assume.assumeTrue("unsupport.", false);
3940
}
4041

42+
@Disabled
4143
@Override
4244
public void testCreatedFileIsVisibleOnFlush() {
43-
Assume.assumeTrue("unsupport", false);
4445
}
4546
}

‎hadoop-cloud-storage-project/hadoop-huaweicloud/src/test/java/org/apache/hadoop/fs/obs/TestOBSContractRename.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import org.apache.hadoop.conf.Configuration;
2222
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
2323
import org.apache.hadoop.fs.contract.AbstractFSContract;
24-
import org.junit.Assume;
24+
import org.junit.jupiter.api.Disabled;
2525

2626
/**
2727
* Rename test cases on obs file system.
@@ -33,13 +33,13 @@ protected AbstractFSContract createContract(final Configuration conf) {
3333
return new OBSContract(conf);
3434
}
3535

36+
@Disabled
3637
@Override
3738
public void testRenameFileUnderFileSubdir() {
38-
Assume.assumeTrue("unsupport.", false);
3939
}
4040

41+
@Disabled
4142
@Override
4243
public void testRenameFileUnderFile() {
43-
Assume.assumeTrue("unsupport.", false);
4444
}
4545
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@
7373
import java.util.concurrent.LinkedBlockingQueue;
7474
import java.util.concurrent.TimeUnit;
7575

76+
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.ECRedundancy.DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED;
77+
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.ECRedundancy.DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED_DEFAILT;
78+
7679
/**
7780
* This class supports writing files in striped layout and erasure coded format.
7881
* Each stripe contains a sequence of cells.
@@ -283,6 +286,7 @@ private void flipDataBuffers() {
283286
private CompletionService<Void> flushAllExecutorCompletionService;
284287
private int blockGroupIndex;
285288
private long datanodeRestartTimeout;
289+
private final int failedBlocksTolerated;
286290

287291
/** Construct a new output stream for creating a file. */
288292
DFSStripedOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat,
@@ -322,6 +326,15 @@ private void flipDataBuffers() {
322326
currentPackets = new DFSPacket[streamers.size()];
323327
datanodeRestartTimeout = dfsClient.getConf().getDatanodeRestartTimeout();
324328
setCurrentStreamer(0);
329+
330+
int failedBlocksToleratedTmp = dfsClient.getConfiguration().getInt(
331+
DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED,
332+
DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED_DEFAILT);
333+
if (failedBlocksToleratedTmp < 0) {
334+
failedBlocksToleratedTmp = ecPolicy.getNumParityUnits();
335+
}
336+
failedBlocksTolerated = Math.min(failedBlocksToleratedTmp,
337+
ecPolicy.getNumParityUnits());
325338
}
326339

327340
/** Construct a new output stream for appending to a file. */
@@ -402,11 +415,11 @@ private Set<StripedDataStreamer> checkStreamers() throws IOException {
402415
LOG.debug("original failed streamers: {}", failedStreamers);
403416
LOG.debug("newly failed streamers: {}", newFailed);
404417
}
405-
if (failCount > (numAllBlocks - numDataBlocks)) {
418+
if (failCount > failedBlocksTolerated) {
406419
closeAllStreamers();
407420
throw new IOException("Failed: the number of failed blocks = "
408-
+ failCount + " > the number of parity blocks = "
409-
+ (numAllBlocks - numDataBlocks));
421+
+ failCount + " > the number of failed blocks tolerated = "
422+
+ failedBlocksTolerated);
410423
}
411424
return newFailed;
412425
}
@@ -687,7 +700,7 @@ private void checkStreamerFailures(boolean isNeedFlushAllPackets)
687700
// 2) create new block outputstream
688701
newFailed = waitCreatingStreamers(healthySet);
689702
if (newFailed.size() + failedStreamers.size() >
690-
numAllBlocks - numDataBlocks) {
703+
failedBlocksTolerated) {
691704
// The write has failed, Close all the streamers.
692705
closeAllStreamers();
693706
throw new IOException(

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,13 @@ interface ByteArrayManager {
427427
PREFIX + "count-reset-time-period-ms";
428428
long COUNT_RESET_TIME_PERIOD_MS_DEFAULT = 10 * MS_PER_SECOND;
429429
}
430+
431+
@SuppressWarnings("checkstyle:InterfaceIsType")
432+
interface ECRedundancy {
433+
String DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED =
434+
"dfs.client.ec.write.failed.blocks.tolerated";
435+
int DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED_DEFAILT = -1;
436+
}
430437
}
431438

432439
/** dfs.client.block.write configuration properties */

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ protected void initChannel(SocketChannel ch) throws Exception {
164164
}
165165
p.addLast(
166166
new ChunkedWriteHandler(),
167-
new URLDispatcher(jettyAddr, conf, confForCreate));
167+
new URLDispatcher(jettyAddr, conf, confForCreate, false));
168168
}
169169
});
170170

@@ -222,7 +222,7 @@ protected void initChannel(SocketChannel ch) throws Exception {
222222
}
223223
p.addLast(
224224
new ChunkedWriteHandler(),
225-
new URLDispatcher(jettyAddr, conf, confForCreate));
225+
new URLDispatcher(jettyAddr, conf, confForCreate, true));
226226
}
227227
});
228228
} else {

0 commit comments

Comments
 (0)