Skip to content

Commit c9725ff

Browse files
steveloughransteveloughran
committed
HADOOP-19696. hadoop binary distribution to move cloud connectors to hadoop common/lib
* new assembly for hadoop cloud storage * hadoop-cloud-storage does the assembly on -Pdist * layout stitching to move into share/hadoop/common/lib * remove connectors from hadoop-tools-dist * cut old jackson version from huawaei cloud dependency -even though it was being upgraded by our own artifacts, it was a complication.
1 parent 3baa288 commit c9725ff

File tree

6 files changed

+127
-40
lines changed

6 files changed

+127
-40
lines changed

‎dev-support/bin/dist-layout-stitching‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,10 @@ run cp -p "${ROOT}/README.txt" .
130130
run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
131131
run copy "${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}" .
132132
run copy "${ROOT}/hadoop-common-project/hadoop-registry/target/hadoop-registry-${VERSION}" .
133+
134+
# cloud connectors go into common
135+
run copy "${ROOT}/hadoop-cloud-storage-project/hadoop-cloud-storage/target/hadoop-cloud-storage-${VERSION}" .
136+
133137
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
134138
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
135139
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-cloud-storage.xml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the Li2cense is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
20+
<id>hadoop-cloud-storage</id>
21+
<formats>
22+
<format>dir</format>
23+
</formats>
24+
<includeBaseDirectory>false</includeBaseDirectory>
25+
26+
<fileSets>
27+
<fileSet>
28+
<directory>../../hadoop-tools/hadoop-aws/src/main/bin</directory>
29+
<outputDirectory>/bin</outputDirectory>
30+
<fileMode>0755</fileMode>
31+
</fileSet>
32+
<fileSet>
33+
<directory>./../hadoop-tools/hadoop-aws/src/main/shellprofile.d</directory>
34+
<includes>
35+
<include>*</include>
36+
</includes>
37+
<outputDirectory>/libexec/shellprofile.d</outputDirectory>
38+
<fileMode>0755</fileMode>
39+
</fileSet>
40+
</fileSets>
41+
42+
<dependencySets>
43+
<dependencySet>
44+
<outputDirectory>/share/hadoop/common/lib</outputDirectory>
45+
<unpack>false</unpack>
46+
<scope>runtime</scope>
47+
<useProjectArtifact>false</useProjectArtifact>
48+
<!-- Stop some needless artifact propagation -->
49+
<excludes>
50+
<exclude>org.apache.hadoop:hadoop-annotations</exclude>
51+
<exclude>org.apache.hadoop.thirdparty:hadoop-shaded-guava</exclude>
52+
</excludes>
53+
</dependencySet>
54+
</dependencySets>
55+
</assembly>

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
<exclude>**/file:/**</exclude>
5858
<exclude>**/SecurityAuth.audit*</exclude>
5959
<exclude>patchprocess/**</exclude>
60+
<exclude>**/auth-keys.xml</exclude>
6061
</excludes>
6162
</fileSet>
6263
</fileSets>

‎hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,12 @@
124124
<groupId>org.apache.hadoop</groupId>
125125
<artifactId>hadoop-huaweicloud</artifactId>
126126
<scope>compile</scope>
127+
<exclusions>
128+
<exclusion>
129+
<groupId>com.fasterxml.jackson</groupId>
130+
<artifactId>*</artifactId>
131+
</exclusion>
132+
</exclusions>
127133
</dependency>
128134
<dependency>
129135
<groupId>org.apache.hadoop</groupId>
@@ -146,4 +152,61 @@
146152
</exclusions>
147153
</dependency>
148154
</dependencies>
155+
156+
<build>
157+
<plugins>
158+
<plugin>
159+
<artifactId>maven-deploy-plugin</artifactId>
160+
<configuration>
161+
<skip>true</skip>
162+
</configuration>
163+
</plugin>
164+
<plugin>
165+
<groupId>org.apache.rat</groupId>
166+
<artifactId>apache-rat-plugin</artifactId>
167+
<configuration>
168+
</configuration>
169+
</plugin>
170+
</plugins>
171+
</build>
172+
<profiles>
173+
<profile>
174+
<id>dist</id>
175+
<activation>
176+
<activeByDefault>false</activeByDefault>
177+
</activation>
178+
<build>
179+
<plugins>
180+
<plugin>
181+
<groupId>org.apache.maven.plugins</groupId>
182+
<artifactId>maven-assembly-plugin</artifactId>
183+
<dependencies>
184+
<dependency>
185+
<groupId>org.apache.hadoop</groupId>
186+
<artifactId>hadoop-assemblies</artifactId>
187+
<version>${project.version}</version>
188+
</dependency>
189+
</dependencies>
190+
<executions>
191+
<execution>
192+
<id>dist</id>
193+
<phase>prepare-package</phase>
194+
<goals>
195+
<goal>single</goal>
196+
</goals>
197+
<configuration>
198+
<appendAssemblyId>false</appendAssemblyId>
199+
<attach>false</attach>
200+
<finalName>${project.artifactId}-${project.version}</finalName>
201+
<descriptorRefs>
202+
<descriptorRef>hadoop-cloud-storage</descriptorRef>
203+
</descriptorRefs>
204+
</configuration>
205+
</execution>
206+
</executions>
207+
</plugin>
208+
</plugins>
209+
</build>
210+
</profile>
211+
</profiles>
149212
</project>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,10 @@
169169
<artifactId>log4j-api</artifactId>
170170
<groupId>org.apache.logging.log4j</groupId>
171171
</exclusion>
172+
<exclusion>
173+
<groupId>com.fasterxml.jackson.core</groupId>
174+
<artifactId>*</artifactId>
175+
</exclusion>
172176
</exclusions>
173177
</dependency>
174178
</dependencies>

‎hadoop-tools/hadoop-tools-dist/pom.xml‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -92,30 +92,12 @@
9292
<type>pom</type>
9393
<version>${project.version}</version>
9494
</dependency>
95-
<dependency>
96-
<groupId>org.apache.hadoop</groupId>
97-
<artifactId>hadoop-aws</artifactId>
98-
<scope>compile</scope>
99-
<version>${project.version}</version>
100-
</dependency>
10195
<dependency>
10296
<groupId>org.apache.hadoop</groupId>
10397
<artifactId>hadoop-kafka</artifactId>
10498
<scope>compile</scope>
10599
<version>${project.version}</version>
106100
</dependency>
107-
<dependency>
108-
<groupId>org.apache.hadoop</groupId>
109-
<artifactId>hadoop-azure</artifactId>
110-
<scope>compile</scope>
111-
<version>${project.version}</version>
112-
</dependency>
113-
<dependency>
114-
<groupId>org.apache.hadoop</groupId>
115-
<artifactId>hadoop-aliyun</artifactId>
116-
<scope>compile</scope>
117-
<version>${project.version}</version>
118-
</dependency>
119101
<dependency>
120102
<groupId>org.apache.hadoop</groupId>
121103
<artifactId>hadoop-sls</artifactId>
@@ -127,34 +109,12 @@
127109
<version>${project.version}</version>
128110
<scope>compile</scope>
129111
</dependency>
130-
<dependency>
131-
<groupId>org.apache.hadoop</groupId>
132-
<artifactId>hadoop-azure-datalake</artifactId>
133-
<scope>compile</scope>
134-
<version>${project.version}</version>
135-
</dependency>
136112
<dependency>
137113
<groupId>org.apache.hadoop</groupId>
138114
<artifactId>hadoop-fs2img</artifactId>
139115
<scope>compile</scope>
140116
<version>${project.version}</version>
141117
</dependency>
142-
<dependency>
143-
<groupId>org.apache.hadoop</groupId>
144-
<artifactId>hadoop-gcp</artifactId>
145-
<scope>compile</scope>
146-
<version>${project.version}</version>
147-
<!--
148-
Exclude transitive dependencies to prevent dependency convergence
149-
problems. hadoop-gcp is a self-contained shaded jar.
150-
-->
151-
<exclusions>
152-
<exclusion>
153-
<groupId>*</groupId>
154-
<artifactId>*</artifactId>
155-
</exclusion>
156-
</exclusions>
157-
</dependency>
158118
</dependencies>
159119

160120
<build>

0 commit comments

Comments
 (0)