Skip to content

Commit 7421ba8

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into HADOOP-18302 
2 parents 27fd624 + 21bae31 commit 7421ba8

File tree

206 files changed

+8074
-1299
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

206 files changed

+8074
-1299
lines changed

‎LICENSE-binary‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ commons-collections:commons-collections:3.2.2
251251
commons-daemon:commons-daemon:1.0.13
252252
commons-io:commons-io:2.8.0
253253
commons-logging:commons-logging:1.1.3
254-
commons-net:commons-net:3.6
254+
commons-net:commons-net:3.8.0
255255
de.ruedigermoeller:fst:2.50
256256
io.grpc:grpc-api:1.26.0
257257
io.grpc:grpc-context:1.26.0
@@ -362,7 +362,7 @@ org.ehcache:ehcache:3.3.1
362362
org.lz4:lz4-java:1.7.1
363363
org.objenesis:objenesis:2.6
364364
org.xerial.snappy:snappy-java:1.0.5
365-
org.yaml:snakeyaml:1.16:
365+
org.yaml:snakeyaml:1.31:
366366
org.wildfly.openssl:wildfly-openssl:1.0.7.Final
367367

368368

‎hadoop-client-modules/hadoop-client-api/pom.xml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,6 @@
9898
<createSourcesJar>true</createSourcesJar>
9999
<shadeSourcesContent>true</shadeSourcesContent>
100100
</configuration>
101-
<dependencies>
102-
<dependency>
103-
<groupId>org.apache.hadoop</groupId>
104-
<artifactId>hadoop-maven-plugins</artifactId>
105-
<version>${project.version}</version>
106-
</dependency>
107-
</dependencies>
108101
<executions>
109102
<execution>
110103
<phase>package</phase>
@@ -254,8 +247,7 @@
254247
</relocation>
255248
</relocations>
256249
<transformers>
257-
<!-- Needed until MSHADE-182 -->
258-
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
250+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
259251
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
260252
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
261253
<resource>NOTICE.txt</resource>

‎hadoop-client-modules/hadoop-client-minicluster/pom.xml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -671,13 +671,6 @@
671671
<plugin>
672672
<groupId>org.apache.maven.plugins</groupId>
673673
<artifactId>maven-shade-plugin</artifactId>
674-
<dependencies>
675-
<dependency>
676-
<groupId>org.apache.hadoop</groupId>
677-
<artifactId>hadoop-maven-plugins</artifactId>
678-
<version>${project.version}</version>
679-
</dependency>
680-
</dependencies>
681674
<executions>
682675
<execution>
683676
<phase>package</phase>
@@ -1052,8 +1045,7 @@
10521045
</relocation>
10531046
</relocations>
10541047
<transformers>
1055-
<!-- Needed until MSHADE-182 -->
1056-
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
1048+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
10571049
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
10581050
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
10591051
<resources>

‎hadoop-client-modules/hadoop-client-runtime/pom.xml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -128,13 +128,6 @@
128128
<plugin>
129129
<groupId>org.apache.maven.plugins</groupId>
130130
<artifactId>maven-shade-plugin</artifactId>
131-
<dependencies>
132-
<dependency>
133-
<groupId>org.apache.hadoop</groupId>
134-
<artifactId>hadoop-maven-plugins</artifactId>
135-
<version>${project.version}</version>
136-
</dependency>
137-
</dependencies>
138131
<executions>
139132
<execution>
140133
<phase>package</phase>
@@ -397,8 +390,7 @@
397390
-->
398391
</relocations>
399392
<transformers>
400-
<!-- Needed until MSHADE-182 -->
401-
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
393+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
402394
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
403395
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
404396
<resources>

‎hadoop-common-project/hadoop-common/pom.xml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,36 @@
841841
</execution>
842842
</executions>
843843
</plugin>
844+
<plugin>
845+
<!--Sets the skip.platformToolsetDetection to true if use.platformToolsetVersion is specified.
846+
This implies that the automatic detection of which platform toolset to use will be skipped
847+
and the one specified with use.platformToolsetVersion will be used.-->
848+
<groupId>org.apache.maven.plugins</groupId>
849+
<artifactId>maven-antrun-plugin</artifactId>
850+
<version>1.8</version>
851+
<executions>
852+
<execution>
853+
<phase>validate</phase>
854+
<goals>
855+
<goal>run</goal>
856+
</goals>
857+
<configuration>
858+
<exportAntProperties>true</exportAntProperties>
859+
<target>
860+
<condition property="skip.platformToolsetDetection" value="true" else="false">
861+
<isset property="use.platformToolsetVersion"/>
862+
</condition>
863+
<!--Unfortunately, Maven doesn't have a way to negate a flag, thus we declare a
864+
property which holds the negated value of skip.platformToolsetDetection.-->
865+
<condition property="skip.platformToolsetDetection.negated" value="false" else="true">
866+
<isset property="use.platformToolsetVersion"/>
867+
</condition>
868+
<echo>Skip platform toolset version detection = ${skip.platformToolsetDetection}</echo>
869+
</target>
870+
</configuration>
871+
</execution>
872+
</executions>
873+
</plugin>
844874
<plugin>
845875
<groupId>org.codehaus.mojo</groupId>
846876
<artifactId>exec-maven-plugin</artifactId>
@@ -852,6 +882,7 @@
852882
<goal>exec</goal>
853883
</goals>
854884
<configuration>
885+
<skip>${skip.platformToolsetDetection}</skip>
855886
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
856887
<arguments>
857888
<argument>${basedir}\src\main\winutils</argument>
@@ -866,6 +897,27 @@
866897
<goal>exec</goal>
867898
</goals>
868899
<configuration>
900+
<skip>${skip.platformToolsetDetection}</skip>
901+
<executable>msbuild</executable>
902+
<arguments>
903+
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
904+
<argument>/nologo</argument>
905+
<argument>/p:Configuration=Release</argument>
906+
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
907+
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
908+
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
909+
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
910+
</arguments>
911+
</configuration>
912+
</execution>
913+
<execution>
914+
<id>compile-ms-winutils-using-build-tools</id>
915+
<phase>compile</phase>
916+
<goals>
917+
<goal>exec</goal>
918+
</goals>
919+
<configuration>
920+
<skip>${skip.platformToolsetDetection.negated}</skip>
869921
<executable>msbuild</executable>
870922
<arguments>
871923
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
@@ -875,6 +927,7 @@
875927
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
876928
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
877929
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
930+
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
878931
</arguments>
879932
</configuration>
880933
</execution>
@@ -885,6 +938,7 @@
885938
<goal>exec</goal>
886939
</goals>
887940
<configuration>
941+
<skip>${skip.platformToolsetDetection}</skip>
888942
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
889943
<arguments>
890944
<argument>${basedir}\src\main\native</argument>
@@ -899,6 +953,35 @@
899953
<goal>exec</goal>
900954
</goals>
901955
<configuration>
956+
<skip>${skip.platformToolsetDetection}</skip>
957+
<executable>msbuild</executable>
958+
<arguments>
959+
<argument>${basedir}/src/main/native/native.sln</argument>
960+
<argument>/nologo</argument>
961+
<argument>/p:Configuration=Release</argument>
962+
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
963+
<argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
964+
<argument>/p:CustomZstdLib=${zstd.lib}</argument>
965+
<argument>/p:CustomZstdInclude=${zstd.include}</argument>
966+
<argument>/p:RequireZstd=${require.zstd}</argument>
967+
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
968+
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
969+
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
970+
<argument>/p:RequireOpenssl=${require.openssl}</argument>
971+
<argument>/p:RequireIsal=${require.isal}</argument>
972+
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
973+
<argument>/p:CustomIsalLib=${isal.lib}</argument>
974+
</arguments>
975+
</configuration>
976+
</execution>
977+
<execution>
978+
<id>compile-ms-native-dll-using-build-tools</id>
979+
<phase>compile</phase>
980+
<goals>
981+
<goal>exec</goal>
982+
</goals>
983+
<configuration>
984+
<skip>${skip.platformToolsetDetection.negated}</skip>
902985
<executable>msbuild</executable>
903986
<arguments>
904987
<argument>${basedir}/src/main/native/native.sln</argument>
@@ -916,6 +999,7 @@
916999
<argument>/p:RequireIsal=${require.isal}</argument>
9171000
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
9181001
<argument>/p:CustomIsalLib=${isal.lib}</argument>
1002+
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
9191003
</arguments>
9201004
</configuration>
9211005
</execution>

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java‎

Lines changed: 64 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ private static class ChecksumFSInputChecker extends FSInputChecker implements
174174
private static final int HEADER_LENGTH = 8;
175175

176176
private int bytesPerSum = 1;
177+
private long fileLen = -1L;
177178

178179
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
179180
throws IOException {
@@ -320,6 +321,18 @@ public static long findChecksumOffset(long dataOffset,
320321
return HEADER_LENGTH + (dataOffset/bytesPerSum) * FSInputChecker.CHECKSUM_SIZE;
321322
}
322323

324+
/**
325+
* Calculate length of file if not already cached.
326+
* @return file length.
327+
* @throws IOException any IOE.
328+
*/
329+
private long getFileLength() throws IOException {
330+
if (fileLen == -1L) {
331+
fileLen = fs.getFileStatus(file).getLen();
332+
}
333+
return fileLen;
334+
}
335+
323336
/**
324337
* Find the checksum ranges that correspond to the given data ranges.
325338
* @param dataRanges the input data ranges, which are assumed to be sorted
@@ -371,13 +384,28 @@ static ByteBuffer checkBytes(ByteBuffer sumsBytes,
371384
IntBuffer sums = sumsBytes.asIntBuffer();
372385
sums.position(offset / FSInputChecker.CHECKSUM_SIZE);
373386
ByteBuffer current = data.duplicate();
374-
int numChunks = data.remaining() / bytesPerSum;
387+
int numFullChunks = data.remaining() / bytesPerSum;
388+
boolean partialChunk = ((data.remaining() % bytesPerSum) != 0);
389+
int totalChunks = numFullChunks;
390+
if (partialChunk) {
391+
totalChunks++;
392+
}
375393
CRC32 crc = new CRC32();
376394
// check each chunk to ensure they match
377-
for(int c = 0; c < numChunks; ++c) {
378-
// set the buffer position and the limit
379-
current.limit((c + 1) * bytesPerSum);
395+
for(int c = 0; c < totalChunks; ++c) {
396+
// set the buffer position to the start of every chunk.
380397
current.position(c * bytesPerSum);
398+
399+
if (c == numFullChunks) {
400+
// During last chunk, there may be less than chunk size
401+
// data preset, so setting the limit accordingly.
402+
int lastIncompleteChunk = data.remaining() % bytesPerSum;
403+
current.limit((c * bytesPerSum) + lastIncompleteChunk);
404+
} else {
405+
// set the buffer limit to end of every chunk.
406+
current.limit((c + 1) * bytesPerSum);
407+
}
408+
381409
// compute the crc
382410
crc.reset();
383411
crc.update(current);
@@ -396,11 +424,34 @@ static ByteBuffer checkBytes(ByteBuffer sumsBytes,
396424
return data;
397425
}
398426

427+
/**
428+
* Validates range parameters.
429+
* In case of CheckSum FS, we already have calculated
430+
* fileLength so failing fast here.
431+
* @param ranges requested ranges.
432+
* @param fileLength length of file.
433+
* @throws EOFException end of file exception.
434+
*/
435+
private void validateRangeRequest(List<? extends FileRange> ranges,
436+
final long fileLength) throws EOFException {
437+
for (FileRange range : ranges) {
438+
VectoredReadUtils.validateRangeRequest(range);
439+
if (range.getOffset() + range.getLength() > fileLength) {
440+
final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s",
441+
range.getOffset(), range.getLength(), file);
442+
LOG.warn(errMsg);
443+
throw new EOFException(errMsg);
444+
}
445+
}
446+
}
447+
399448
@Override
400449
public void readVectored(List<? extends FileRange> ranges,
401450
IntFunction<ByteBuffer> allocate) throws IOException {
451+
final long length = getFileLength();
452+
validateRangeRequest(ranges, length);
453+
402454
// If the stream doesn't have checksums, just delegate.
403-
VectoredReadUtils.validateVectoredReadRanges(ranges);
404455
if (sums == null) {
405456
datas.readVectored(ranges, allocate);
406457
return;
@@ -410,15 +461,18 @@ public void readVectored(List<? extends FileRange> ranges,
410461
List<CombinedFileRange> dataRanges =
411462
VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(ranges)), bytesPerSum,
412463
minSeek, maxReadSizeForVectorReads());
464+
// While merging the ranges above, they are rounded up based on the value of bytesPerSum
465+
// which leads to some ranges crossing the EOF thus they need to be fixed else it will
466+
// cause EOFException during actual reads.
467+
for (CombinedFileRange range : dataRanges) {
468+
if (range.getOffset() + range.getLength() > length) {
469+
range.setLength((int) (length - range.getOffset()));
470+
}
471+
}
413472
List<CombinedFileRange> checksumRanges = findChecksumRanges(dataRanges,
414473
bytesPerSum, minSeek, maxSize);
415474
sums.readVectored(checksumRanges, allocate);
416475
datas.readVectored(dataRanges, allocate);
417-
// Data read is correct. I have verified content of dataRanges.
418-
// There is some bug below here as test (testVectoredReadMultipleRanges)
419-
// is failing, should be
420-
// somewhere while slicing the merged data into smaller user ranges.
421-
// Spend some time figuring out but it is a complex code.
422476
for(CombinedFileRange checksumRange: checksumRanges) {
423477
for(FileRange dataRange: checksumRange.getUnderlying()) {
424478
// when we have both the ranges, validate the checksum

0 commit comments

Comments
 (0)