Skip to content

Commit 1f6827f

Browse files
anmolanmol1234anmolanmol1234
authored
Merge branch 'apache:trunk' into HADOOP-18325 
2 parents 40868de + 062c50d commit 1f6827f

File tree

607 files changed

+27430
-18151
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

607 files changed

+27430
-18151
lines changed

‎LICENSE-binary‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ commons-collections:commons-collections:3.2.2
251251
commons-daemon:commons-daemon:1.0.13
252252
commons-io:commons-io:2.8.0
253253
commons-logging:commons-logging:1.1.3
254-
commons-net:commons-net:3.6
254+
commons-net:commons-net:3.8.0
255255
de.ruedigermoeller:fst:2.50
256256
io.grpc:grpc-api:1.26.0
257257
io.grpc:grpc-context:1.26.0
@@ -305,12 +305,12 @@ net.minidev:json-smart:2.4.7
305305
org.apache.avro:avro:1.9.2
306306
org.apache.commons:commons-collections4:4.2
307307
org.apache.commons:commons-compress:1.21
308-
org.apache.commons:commons-configuration2:2.1.1
308+
org.apache.commons:commons-configuration2:2.8.0
309309
org.apache.commons:commons-csv:1.0
310310
org.apache.commons:commons-digester:1.8.1
311311
org.apache.commons:commons-lang3:3.12.0
312312
org.apache.commons:commons-math3:3.6.1
313-
org.apache.commons:commons-text:1.4
313+
org.apache.commons:commons-text:1.9
314314
org.apache.commons:commons-validator:1.6
315315
org.apache.curator:curator-client:5.2.0
316316
org.apache.curator:curator-framework:5.2.0
@@ -325,21 +325,21 @@ org.apache.htrace:htrace-core4:4.1.0-incubating
325325
org.apache.httpcomponents:httpclient:4.5.6
326326
org.apache.httpcomponents:httpcore:4.4.10
327327
org.apache.kafka:kafka-clients:2.8.1
328-
org.apache.kerby:kerb-admin:1.0.1
329-
org.apache.kerby:kerb-client:1.0.1
330-
org.apache.kerby:kerb-common:1.0.1
331-
org.apache.kerby:kerb-core:1.0.1
332-
org.apache.kerby:kerb-crypto:1.0.1
333-
org.apache.kerby:kerb-identity:1.0.1
334-
org.apache.kerby:kerb-server:1.0.1
335-
org.apache.kerby:kerb-simplekdc:1.0.1
336-
org.apache.kerby:kerb-util:1.0.1
337-
org.apache.kerby:kerby-asn1:1.0.1
338-
org.apache.kerby:kerby-config:1.0.1
339-
org.apache.kerby:kerby-pkix:1.0.1
340-
org.apache.kerby:kerby-util:1.0.1
341-
org.apache.kerby:kerby-xdr:1.0.1
342-
org.apache.kerby:token-provider:1.0.1
328+
org.apache.kerby:kerb-admin:2.0.2
329+
org.apache.kerby:kerb-client:2.0.2
330+
org.apache.kerby:kerb-common:2.0.2
331+
org.apache.kerby:kerb-core:2.0.2
332+
org.apache.kerby:kerb-crypto:2.0.2
333+
org.apache.kerby:kerb-identity:2.0.2
334+
org.apache.kerby:kerb-server:2.0.2
335+
org.apache.kerby:kerb-simplekdc:2.0.2
336+
org.apache.kerby:kerb-util:2.0.2
337+
org.apache.kerby:kerby-asn1:2.0.2
338+
org.apache.kerby:kerby-config:2.0.2
339+
org.apache.kerby:kerby-pkix:2.0.2
340+
org.apache.kerby:kerby-util:2.0.2
341+
org.apache.kerby:kerby-xdr:2.0.2
342+
org.apache.kerby:token-provider:2.0.2
343343
org.apache.solr:solr-solrj:8.8.2
344344
org.apache.yetus:audience-annotations:0.5.0
345345
org.apache.zookeeper:zookeeper:3.6.3
@@ -362,7 +362,7 @@ org.ehcache:ehcache:3.3.1
362362
org.lz4:lz4-java:1.7.1
363363
org.objenesis:objenesis:2.6
364364
org.xerial.snappy:snappy-java:1.0.5
365-
org.yaml:snakeyaml:1.16:
365+
org.yaml:snakeyaml:1.32
366366
org.wildfly.openssl:wildfly-openssl:1.0.7.Final
367367

368368

@@ -523,7 +523,7 @@ junit:junit:4.13.2
523523
HSQL License
524524
------------
525525

526-
org.hsqldb:hsqldb:2.3.4
526+
org.hsqldb:hsqldb:2.5.2
527527

528528

529529
JDOM License

‎hadoop-client-modules/hadoop-client-api/pom.xml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,6 @@
9898
<createSourcesJar>true</createSourcesJar>
9999
<shadeSourcesContent>true</shadeSourcesContent>
100100
</configuration>
101-
<dependencies>
102-
<dependency>
103-
<groupId>org.apache.hadoop</groupId>
104-
<artifactId>hadoop-maven-plugins</artifactId>
105-
<version>${project.version}</version>
106-
</dependency>
107-
</dependencies>
108101
<executions>
109102
<execution>
110103
<phase>package</phase>
@@ -254,8 +247,7 @@
254247
</relocation>
255248
</relocations>
256249
<transformers>
257-
<!-- Needed until MSHADE-182 -->
258-
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
250+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
259251
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
260252
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
261253
<resource>NOTICE.txt</resource>

‎hadoop-client-modules/hadoop-client-minicluster/pom.xml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -671,13 +671,6 @@
671671
<plugin>
672672
<groupId>org.apache.maven.plugins</groupId>
673673
<artifactId>maven-shade-plugin</artifactId>
674-
<dependencies>
675-
<dependency>
676-
<groupId>org.apache.hadoop</groupId>
677-
<artifactId>hadoop-maven-plugins</artifactId>
678-
<version>${project.version}</version>
679-
</dependency>
680-
</dependencies>
681674
<executions>
682675
<execution>
683676
<phase>package</phase>
@@ -1052,8 +1045,7 @@
10521045
</relocation>
10531046
</relocations>
10541047
<transformers>
1055-
<!-- Needed until MSHADE-182 -->
1056-
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
1048+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
10571049
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
10581050
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
10591051
<resources>

‎hadoop-client-modules/hadoop-client-runtime/pom.xml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -128,13 +128,6 @@
128128
<plugin>
129129
<groupId>org.apache.maven.plugins</groupId>
130130
<artifactId>maven-shade-plugin</artifactId>
131-
<dependencies>
132-
<dependency>
133-
<groupId>org.apache.hadoop</groupId>
134-
<artifactId>hadoop-maven-plugins</artifactId>
135-
<version>${project.version}</version>
136-
</dependency>
137-
</dependencies>
138131
<executions>
139132
<execution>
140133
<phase>package</phase>
@@ -397,8 +390,7 @@
397390
-->
398391
</relocations>
399392
<transformers>
400-
<!-- Needed until MSHADE-182 -->
401-
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
393+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
402394
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
403395
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
404396
<resources>

‎hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,6 @@
127127
<artifactId>hadoop-azure-datalake</artifactId>
128128
<scope>compile</scope>
129129
</dependency>
130-
<dependency>
131-
<groupId>org.apache.hadoop</groupId>
132-
<artifactId>hadoop-openstack</artifactId>
133-
<scope>compile</scope>
134-
</dependency>
135130
<dependency>
136131
<groupId>org.apache.hadoop</groupId>
137132
<artifactId>hadoop-cos</artifactId>

‎hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -379,21 +379,6 @@
379379
<Bug code="JLM" />
380380
</Match>
381381

382-
<!--
383-
OpenStack Swift FS module -closes streams in a different method
384-
from where they are opened.
385-
-->
386-
<Match>
387-
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
388-
<Method name="uploadFileAttempt"/>
389-
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
390-
</Match>
391-
<Match>
392-
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
393-
<Method name="uploadFilePartAttempt"/>
394-
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
395-
</Match>
396-
397382
<!-- code from maven source, null value is checked at callee side. -->
398383
<Match>
399384
<Class name="org.apache.hadoop.util.ComparableVersion$ListItem" />

‎hadoop-common-project/hadoop-common/pom.xml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,36 @@
841841
</execution>
842842
</executions>
843843
</plugin>
844+
<plugin>
845+
<!--Sets the skip.platformToolsetDetection to true if use.platformToolsetVersion is specified.
846+
This implies that the automatic detection of which platform toolset to use will be skipped
847+
and the one specified with use.platformToolsetVersion will be used.-->
848+
<groupId>org.apache.maven.plugins</groupId>
849+
<artifactId>maven-antrun-plugin</artifactId>
850+
<version>1.8</version>
851+
<executions>
852+
<execution>
853+
<phase>validate</phase>
854+
<goals>
855+
<goal>run</goal>
856+
</goals>
857+
<configuration>
858+
<exportAntProperties>true</exportAntProperties>
859+
<target>
860+
<condition property="skip.platformToolsetDetection" value="true" else="false">
861+
<isset property="use.platformToolsetVersion"/>
862+
</condition>
863+
<!--Unfortunately, Maven doesn't have a way to negate a flag, thus we declare a
864+
property which holds the negated value of skip.platformToolsetDetection.-->
865+
<condition property="skip.platformToolsetDetection.negated" value="false" else="true">
866+
<isset property="use.platformToolsetVersion"/>
867+
</condition>
868+
<echo>Skip platform toolset version detection = ${skip.platformToolsetDetection}</echo>
869+
</target>
870+
</configuration>
871+
</execution>
872+
</executions>
873+
</plugin>
844874
<plugin>
845875
<groupId>org.codehaus.mojo</groupId>
846876
<artifactId>exec-maven-plugin</artifactId>
@@ -852,6 +882,7 @@
852882
<goal>exec</goal>
853883
</goals>
854884
<configuration>
885+
<skip>${skip.platformToolsetDetection}</skip>
855886
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
856887
<arguments>
857888
<argument>${basedir}\src\main\winutils</argument>
@@ -866,6 +897,27 @@
866897
<goal>exec</goal>
867898
</goals>
868899
<configuration>
900+
<skip>${skip.platformToolsetDetection}</skip>
901+
<executable>msbuild</executable>
902+
<arguments>
903+
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
904+
<argument>/nologo</argument>
905+
<argument>/p:Configuration=Release</argument>
906+
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
907+
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
908+
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
909+
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
910+
</arguments>
911+
</configuration>
912+
</execution>
913+
<execution>
914+
<id>compile-ms-winutils-using-build-tools</id>
915+
<phase>compile</phase>
916+
<goals>
917+
<goal>exec</goal>
918+
</goals>
919+
<configuration>
920+
<skip>${skip.platformToolsetDetection.negated}</skip>
869921
<executable>msbuild</executable>
870922
<arguments>
871923
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
@@ -875,6 +927,7 @@
875927
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
876928
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
877929
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
930+
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
878931
</arguments>
879932
</configuration>
880933
</execution>
@@ -885,6 +938,7 @@
885938
<goal>exec</goal>
886939
</goals>
887940
<configuration>
941+
<skip>${skip.platformToolsetDetection}</skip>
888942
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
889943
<arguments>
890944
<argument>${basedir}\src\main\native</argument>
@@ -899,6 +953,35 @@
899953
<goal>exec</goal>
900954
</goals>
901955
<configuration>
956+
<skip>${skip.platformToolsetDetection}</skip>
957+
<executable>msbuild</executable>
958+
<arguments>
959+
<argument>${basedir}/src/main/native/native.sln</argument>
960+
<argument>/nologo</argument>
961+
<argument>/p:Configuration=Release</argument>
962+
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
963+
<argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
964+
<argument>/p:CustomZstdLib=${zstd.lib}</argument>
965+
<argument>/p:CustomZstdInclude=${zstd.include}</argument>
966+
<argument>/p:RequireZstd=${require.zstd}</argument>
967+
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
968+
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
969+
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
970+
<argument>/p:RequireOpenssl=${require.openssl}</argument>
971+
<argument>/p:RequireIsal=${require.isal}</argument>
972+
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
973+
<argument>/p:CustomIsalLib=${isal.lib}</argument>
974+
</arguments>
975+
</configuration>
976+
</execution>
977+
<execution>
978+
<id>compile-ms-native-dll-using-build-tools</id>
979+
<phase>compile</phase>
980+
<goals>
981+
<goal>exec</goal>
982+
</goals>
983+
<configuration>
984+
<skip>${skip.platformToolsetDetection.negated}</skip>
902985
<executable>msbuild</executable>
903986
<arguments>
904987
<argument>${basedir}/src/main/native/native.sln</argument>
@@ -916,6 +999,7 @@
916999
<argument>/p:RequireIsal=${require.isal}</argument>
9171000
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
9181001
<argument>/p:CustomIsalLib=${isal.lib}</argument>
1002+
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
9191003
</arguments>
9201004
</configuration>
9211005
</execution>

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ static void writeResponse(Configuration conf,
9898
if (FORMAT_JSON.equals(format)) {
9999
Configuration.dumpConfiguration(conf, propertyName, out);
100100
} else if (FORMAT_XML.equals(format)) {
101-
conf.writeXml(propertyName, out);
101+
conf.writeXml(propertyName, out, conf);
102102
} else {
103103
throw new BadFormatException("Bad format: " + format);
104104
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
public class ConfigRedactor {
3838

3939
private static final String REDACTED_TEXT = "<redacted>";
40+
private static final String REDACTED_XML = "******";
4041

4142
private List<Pattern> compiledPatterns;
4243

@@ -84,4 +85,19 @@ private boolean configIsSensitive(String key) {
8485
}
8586
return false;
8687
}
88+
89+
/**
90+
* Given a key / value pair, decides whether or not to redact and returns
91+
* either the original value or text indicating it has been redacted.
92+
*
93+
* @param key param key.
94+
* @param value param value, will return if conditions permit.
95+
* @return Original value, or text indicating it has been redacted
96+
*/
97+
public String redactXml(String key, String value) {
98+
if (configIsSensitive(key)) {
99+
return REDACTED_XML;
100+
}
101+
return value;
102+
}
87103
}

0 commit comments

Comments
 (0)