Skip to content

Commit 8fdbd8c

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into YARN-11267-V2
2 parents 8aa05f7 + 72939fe commit 8fdbd8c

File tree

585 files changed

+14011
-9019
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

585 files changed

+14011
-9019
lines changed

‎BUILDING.txt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -635,11 +635,12 @@ hadoop-dist/target upon successful build. Run these commands from an
635635
"x64 Native Tools Command Prompt for VS 2019" which can be found under "Visual Studio 2019" in the
636636
Windows start menu. If you're using the Docker image from Dockerfile_windows_10, you'll be
637637
logged into "x64 Native Tools Command Prompt for VS 2019" automatically when you start the
638-
container.
638+
container. The Docker image does not have a full VS install, so you need to add the
639+
-Dskip.platformToolsetDetection option (already included below in the examples).
639640

640641
> set classpath=
641642
> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
642-
> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist^
643+
> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist -Dskip.platformToolsetDetection^
643644
-Drequire.openssl -Drequire.test.libhadoop -Pyarn-ui -Dshell-executable=C:\Git\bin\bash.exe^
644645
-Dtar -Dopenssl.prefix=C:\vcpkg\installed\x64-windows^
645646
-Dcmake.prefix.path=C:\vcpkg\installed\x64-windows^
@@ -651,7 +652,7 @@ Assuming that we're still running in the Docker container hadoop-windows-10-buil
651652
following command to create the Apache Hadoop release tarball -
652653

653654
> set IS_WINDOWS=1
654-
> set MVN_ARGS="-Dshell-executable=C:\Git\bin\bash.exe -Dhttps.protocols=TLSv1.2 -Pnative-win -Drequire.openssl -Dopenssl.prefix=C:\vcpkg\installed\x64-windows -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo -Dwindows.build.hdfspp.dll=off -Duse.platformToolsetVersion=v142 -Dwindows.no.sasl=on -DskipTests -DskipDocs -Drequire.test.libhadoop"
655+
> set MVN_ARGS="-Dshell-executable=C:\Git\bin\bash.exe -Dhttps.protocols=TLSv1.2 -Pnative-win -Dskip.platformToolsetDetection -Drequire.openssl -Dopenssl.prefix=C:\vcpkg\installed\x64-windows -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo -Dwindows.build.hdfspp.dll=off -Duse.platformToolsetVersion=v142 -Dwindows.no.sasl=on -DskipTests -DskipDocs -Drequire.test.libhadoop"
655656
> C:\Git\bin\bash.exe C:\hadoop\dev-support\bin\create-release --mvnargs=%MVN_ARGS%
656657

657658
Note:

‎LICENSE-binary‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -218,12 +218,12 @@ com.aliyun:aliyun-java-sdk-sts:3.0.0
218218
com.aliyun.oss:aliyun-sdk-oss:3.13.2
219219
com.cedarsoftware:java-util:1.9.0
220220
com.cedarsoftware:json-io:2.5.1
221-
com.fasterxml.jackson.core:jackson-annotations:2.12.7
222-
com.fasterxml.jackson.core:jackson-core:2.12.7
223-
com.fasterxml.jackson.core:jackson-databind:2.12.7.1
224-
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
225-
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
226-
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
221+
com.fasterxml.jackson.core:jackson-annotations:2.14.3
222+
com.fasterxml.jackson.core:jackson-core:2.14.3
223+
com.fasterxml.jackson.core:jackson-databind:2.14.3
224+
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.3
225+
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.3
226+
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.3
227227
com.fasterxml.uuid:java-uuid-generator:3.1.4
228228
com.fasterxml.woodstox:woodstox-core:5.4.0
229229
com.github.ben-manes.caffeine:caffeine:2.9.3
@@ -298,12 +298,11 @@ javax.inject:javax.inject:1
298298
net.java.dev.jna:jna:5.2.0
299299
net.minidev:accessors-smart:1.2
300300
org.apache.avro:avro:1.11.4
301-
org.apache.avro:avro:1.11.3
302301
org.apache.commons:commons-compress:1.26.1
303302
org.apache.commons:commons-configuration2:2.10.1
304303
org.apache.commons:commons-csv:1.9.0
305304
org.apache.commons:commons-digester:1.8.1
306-
org.apache.commons:commons-lang3:3.12.0
305+
org.apache.commons:commons-lang3:3.17.0
307306
org.apache.commons:commons-math3:3.6.1
308307
org.apache.commons:commons-text:1.10.0
309308
org.apache.commons:commons-validator:1.6
@@ -363,7 +362,7 @@ org.xerial.snappy:snappy-java:1.1.10.4
363362
org.yaml:snakeyaml:2.0
364363
org.wildfly.openssl:wildfly-openssl:2.1.4.Final
365364
ro.isdc.wro4j:wro4j-maven-plugin:1.8.0
366-
software.amazon.awssdk:bundle:2.25.53
365+
software.amazon.awssdk:bundle:2.29.52
367366
net.jodah:failsafe:2.4.4
368367

369368
--------------------------------------------------------------------------------
@@ -505,8 +504,7 @@ javax.cache:cache-api:1.1.1
505504
javax.servlet:javax.servlet-api:3.1.0
506505
javax.servlet.jsp:jsp-api:2.1
507506
javax.websocket:javax.websocket-api:1.0
508-
javax.ws.rs:jsr311-api:1.1.1
509-
javax.xml.bind:jaxb-api:2.2.11
507+
javax.xml.bind:jaxb-api:2.3.1
510508

511509
Eclipse Distribution License (EDL) 1.0
512510
--------------------------

‎dev-support/docker/Dockerfile_windows_10‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,20 +49,13 @@ RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\G
4949
# hadolint ignore=DL3003
5050
RUN powershell git clone https://github.com/microsoft/vcpkg.git \
5151
&& cd vcpkg \
52-
&& git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
52+
&& git fetch --all \
53+
&& git checkout 2025.03.19 \
5354
&& .\bootstrap-vcpkg.bat
5455

55-
# Fix wrong download link for 7-zip in .\vcpkg\scripts\vcpkgTools.xml:
56-
# - https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0 (not working anymore) is replaced with
57-
# - https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0 (working)
58-
# Replacing is done using the Linux tool "sed".
59-
RUN choco install sed -y
60-
RUN sed -i "s,https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0,https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0,g" .\vcpkg\scripts\vcpkgTools.xml
61-
62-
RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
63-
RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
64-
RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
65-
RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
56+
ADD vcpkg/vcpkg.json .
57+
58+
RUN powershell .\vcpkg\vcpkg.exe install --x-install-root .\vcpkg\installed
6659

6760
# Install Azul Java 8 JDK.
6861
RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip

‎dev-support/docker/vcpkg/vcpkg.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
3+
"dependencies": [
4+
"boost",
5+
"protobuf",
6+
"openssl",
7+
"zlib"
8+
],
9+
"builtin-baseline": "289a69379604112a433874fe8b9812dad3103341",
10+
"overrides": [
11+
{
12+
"name": "protobuf",
13+
"version": "3.21.12"
14+
}
15+
]
16+
}

‎hadoop-client-modules/hadoop-client-integration-tests/pom.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,26 @@
4747
<artifactId>junit</artifactId>
4848
<scope>test</scope>
4949
</dependency>
50+
<dependency>
51+
<groupId>org.junit.jupiter</groupId>
52+
<artifactId>junit-jupiter-api</artifactId>
53+
<scope>test</scope>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.junit.jupiter</groupId>
57+
<artifactId>junit-jupiter-engine</artifactId>
58+
<scope>test</scope>
59+
</dependency>
60+
<dependency>
61+
<groupId>org.junit.jupiter</groupId>
62+
<artifactId>junit-jupiter-params</artifactId>
63+
<scope>test</scope>
64+
</dependency>
65+
<dependency>
66+
<groupId>org.junit.platform</groupId>
67+
<artifactId>junit-platform-launcher</artifactId>
68+
<scope>test</scope>
69+
</dependency>
5070
<dependency>
5171
<groupId>org.lz4</groupId>
5272
<artifactId>lz4-java</artifactId>

‎hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseHadoopCodecs.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020

2121
package org.apache.hadoop.example;
2222

23-
import static org.junit.Assert.assertEquals;
24-
import static org.junit.Assert.assertFalse;
23+
import static org.junit.jupiter.api.Assertions.assertEquals;
24+
import static org.junit.jupiter.api.Assertions.assertFalse;
2525

2626
import java.io.*;
2727
import java.util.Arrays;
@@ -37,7 +37,7 @@
3737
import org.apache.hadoop.io.compress.CompressionOutputStream;
3838
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
3939
import org.apache.hadoop.util.ReflectionUtils;
40-
import org.junit.Test;
40+
import org.junit.jupiter.api.Test;
4141
import org.slf4j.Logger;
4242
import org.slf4j.LoggerFactory;
4343

@@ -134,8 +134,8 @@ private void codecTest(Configuration conf, int seed, int count, String codecClas
134134
int expected;
135135
do {
136136
expected = originalIn.read();
137-
assertEquals("Inflated stream read by byte does not match",
138-
expected, inflateFilter.read());
137+
assertEquals(expected, inflateFilter.read(),
138+
"Inflated stream read by byte does not match");
139139
} while (expected != -1);
140140
}
141141

‎hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,14 @@
2020

2121
package org.apache.hadoop.example;
2222

23+
import static org.junit.jupiter.api.Assertions.assertEquals;
24+
2325
import java.io.IOException;
2426
import java.net.URISyntaxException;
2527

26-
import org.junit.After;
27-
import org.junit.Assert;
28-
import org.junit.Before;
29-
import org.junit.Test;
28+
import org.junit.jupiter.api.AfterEach;
29+
import org.junit.jupiter.api.BeforeEach;
30+
import org.junit.jupiter.api.Test;
3031

3132
import org.slf4j.Logger;
3233
import org.slf4j.LoggerFactory;
@@ -69,7 +70,7 @@ public class ITUseMiniCluster {
6970
+ "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident,"
7071
+ " sunt in culpa qui officia deserunt mollit anim id est laborum.";
7172

72-
@Before
73+
@BeforeEach
7374
public void clusterUp() throws IOException {
7475
final Configuration conf = new HdfsConfiguration();
7576
cluster = new MiniDFSCluster.Builder(conf)
@@ -84,7 +85,7 @@ public void clusterUp() throws IOException {
8485
yarnCluster.start();
8586
}
8687

87-
@After
88+
@AfterEach
8889
public void clusterDown() {
8990
if (cluster != null) {
9091
cluster.close();
@@ -111,7 +112,7 @@ public void simpleReadAfterWrite(final FileSystem fs) throws IOException {
111112
}
112113
try (FSDataInputStream in = fs.open(path)) {
113114
final String result = in.readUTF();
114-
Assert.assertEquals("Didn't read back text we wrote.", TEXT, result);
115+
assertEquals(TEXT, result, "Didn't read back text we wrote.");
115116
}
116117
}
117118

‎hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,16 @@
2020
import com.qcloud.cos.auth.COSCredentials;
2121
import com.qcloud.cos.auth.COSCredentialsProvider;
2222
import org.apache.hadoop.conf.Configuration;
23-
import org.junit.Test;
23+
import org.junit.jupiter.api.Test;
2424
import org.slf4j.Logger;
2525
import org.slf4j.LoggerFactory;
2626

2727
import java.io.IOException;
2828
import java.net.URI;
2929
import java.net.URISyntaxException;
3030

31-
import static org.junit.Assert.assertNotNull;
32-
import static org.junit.Assert.fail;
31+
import static org.junit.jupiter.api.Assertions.assertNotNull;
32+
import static org.junit.jupiter.api.Assertions.fail;
3333

3434
public class TestCosCredentials {
3535
private static final Logger LOG =
@@ -76,7 +76,7 @@ private void validateCredentials(URI uri, Configuration configuration)
7676
COSCredentialsProvider credentialsProvider =
7777
CosNUtils.createCosCredentialsProviderSet(uri, configuration);
7878
COSCredentials cosCredentials = credentialsProvider.getCredentials();
79-
assertNotNull("The cos credentials obtained is null.", cosCredentials);
79+
assertNotNull(cosCredentials, "The cos credentials obtained is null.");
8080
if (configuration.get(
8181
CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase(
8282
"org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider")

‎hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosNInputStream.java‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,14 @@
2323
import org.apache.hadoop.fs.Path;
2424
import org.apache.hadoop.fs.contract.ContractTestUtils;
2525
import org.apache.hadoop.io.IOUtils;
26-
import org.junit.*;
26+
import org.junit.jupiter.api.AfterEach;
27+
import org.junit.jupiter.api.BeforeEach;
28+
import org.junit.jupiter.api.Disabled;
29+
import org.junit.jupiter.api.Test;
2730
import org.slf4j.Logger;
2831
import org.slf4j.LoggerFactory;
2932

30-
import static org.junit.Assert.assertTrue;
33+
import static org.junit.jupiter.api.Assertions.assertTrue;
3134

3235
import java.io.IOException;
3336
import java.util.Random;
@@ -43,15 +46,15 @@ public class TestCosNInputStream {
4346

4447
private Path testRootDir;
4548

46-
@Before
49+
@BeforeEach
4750
public void setUp() throws IOException {
4851
Configuration configuration = new Configuration();
4952
this.fs = CosNTestUtils.createTestFileSystem(configuration);
5053
this.testRootDir = CosNTestUtils.createTestPath(new Path("/test"));
5154
LOG.info("test root dir: " + this.testRootDir);
5255
}
5356

54-
@After
57+
@AfterEach
5558
public void tearDown() throws IOException {
5659
if (null != this.fs) {
5760
this.fs.delete(this.testRootDir, true);
@@ -76,9 +79,9 @@ public void testSeek() throws Exception {
7679
for (int i = 0; i != seekTimes; i++) {
7780
long pos = fileSize / (seekTimes - i) - 1;
7881
inputStream.seek(pos);
79-
assertTrue("expected position at: " +
80-
pos + ", but got: " + inputStream.getPos(),
81-
inputStream.getPos() == pos);
82+
assertTrue(inputStream.getPos() == pos,
83+
"expected position at: " +
84+
pos + ", but got: " + inputStream.getPos());
8285
LOG.info("completed seeking at pos: " + inputStream.getPos());
8386
}
8487
LOG.info("begin to random position seeking test...");
@@ -87,9 +90,9 @@ public void testSeek() throws Exception {
8790
long pos = Math.abs(random.nextLong()) % fileSize;
8891
LOG.info("seeking for pos: " + pos);
8992
inputStream.seek(pos);
90-
assertTrue("expected position at: " +
91-
pos + ", but got: " + inputStream.getPos(),
92-
inputStream.getPos() == pos);
93+
assertTrue(inputStream.getPos() == pos,
94+
"expected position at: " +
95+
pos + ", but got: " + inputStream.getPos());
9396
LOG.info("completed seeking at pos: " + inputStream.getPos());
9497
}
9598
}
@@ -110,16 +113,16 @@ public void testGetPos() throws Exception {
110113
Random random = new Random();
111114
long pos = Math.abs(random.nextLong()) % fileSize;
112115
inputStream.seek(pos);
113-
assertTrue("expected position at: " +
114-
pos + ", but got: " + inputStream.getPos(),
115-
inputStream.getPos() == pos);
116+
assertTrue(inputStream.getPos() == pos,
117+
"expected position at: " +
118+
pos + ", but got: " + inputStream.getPos());
116119
LOG.info("completed get pos tests.");
117120
}
118121

119122
/**
120123
* Method: seekToNewSource(long targetPos).
121124
*/
122-
@Ignore("Not ready yet")
125+
@Disabled("Not ready yet")
123126
public void testSeekToNewSource() throws Exception {
124127
LOG.info("Currently it is not supported to " +
125128
"seek the offset in a new source.");
@@ -154,8 +157,9 @@ public void testRead() throws Exception {
154157

155158
if (bytesRead % (1 * Unit.MB) == 0) {
156159
int available = inputStream.available();
157-
assertTrue("expected remaining: " + (fileSize - bytesRead) +
158-
" but got: " + available, (fileSize - bytesRead) == available);
160+
assertTrue((fileSize - bytesRead) == available,
161+
"expected remaining: " + (fileSize - bytesRead) +
162+
" but got: " + available);
159163
LOG.info("Bytes read: " +
160164
Math.round((double) bytesRead / Unit.MB) + "MB");
161165
}

‎hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosNOutputStream.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,10 @@
2121
import org.apache.hadoop.fs.FileSystem;
2222
import org.apache.hadoop.fs.Path;
2323
import org.apache.hadoop.fs.contract.ContractTestUtils;
24-
import org.junit.Rule;
25-
import org.junit.Test;
26-
import org.junit.Before;
27-
import org.junit.After;
28-
import org.junit.rules.Timeout;
24+
import org.junit.jupiter.api.AfterEach;
25+
import org.junit.jupiter.api.BeforeEach;
26+
import org.junit.jupiter.api.Test;
27+
import org.junit.jupiter.api.Timeout;
2928

3029
import java.io.IOException;
3130

@@ -34,14 +33,12 @@
3433
* <p>
3534
* If the test.fs.cosn.name property is not set, all test case will fail.
3635
*/
36+
@Timeout(3600)
3737
public class TestCosNOutputStream {
3838
private FileSystem fs;
3939
private Path testRootDir;
4040

41-
@Rule
42-
public Timeout timeout = new Timeout(3600 * 1000);
43-
44-
@Before
41+
@BeforeEach
4542
public void setUp() throws Exception {
4643
Configuration configuration = new Configuration();
4744
configuration.setInt(
@@ -53,7 +50,7 @@ public void setUp() throws Exception {
5350
this.testRootDir = new Path("/test");
5451
}
5552

56-
@After
53+
@AfterEach
5754
public void tearDown() throws Exception {
5855
}
5956

0 commit comments

Comments
 (0)