HADOOP-19696. module hadoop-cloud-storage-dist for the distribution

This ensures that anything done dependency-wise for packaging
doesn't impact the hadoop-cloud-storage module and any downstream
uses.

- separate profile for each component to pull in all dependencies
- hadoop-azure is always included, hadoop-aws *except* bundle.jar
- hadoop-gcp and hadoop-tos are complete iff shaded
- updated BUILDING.txt

This is enough to let anyone cut a release with their choice of functional
cloud connectors.

Author: steveloughran

BUILDING.txt

@@ -388,6 +388,51 @@ Create a local staging version of the website (in /tmp/hadoop-site)
 
 Note that the site needs to be built in a second pass after other artifacts.
 
+----------------------------------------------------------------------------------
+Including Cloud Connector Dependencies in Distributions:
+
+Hadoop distributions include the hadoop modules to work with data and services
+on cloud infrastructure
+
+However, dependencies are omitted for all cloud connectors except hadoop-azure
+(abfs:// and wasb://) and possibly hadoop-gcp (gs://) and hadoop-tos (tos://).
+For the latter two modules, it depends on shading options.
+
+For hadoop-aws the AWS SDK bundle.jar omitted, but everything else is included.
+
+* This keeps binary release size below the limit of apache distributions
+* Reduces download and size overhead in docker usage.
+* Reduces the CVE attack surface
+* Reduces the risk of classpath conflict.
+
+To produce a build with the specific desired dependencies, the build must be executed
+with the relevant profile of ${module}-package.
+
+For example, a build with the hadoop-aws and hadoop-azure-datalake dependencies,
+run with
+
+ mvn package -Pdist -DskipTests -Dhadoop-aws-package -Dhadoop-azure-datalake-package
+
+Available package profiles:
+  hadoop-aliyun-package
+  hadoop-aws-package
+  hadoop-azure-datalake-package
+  hadoop-cos-package
+  hadoop-gcp-package
+  hadoop-huaweicloud-package
+  hadoop-tos-package
+
+To build a complete distribution then with all cloud dependencies included:
+
+mvn package -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true \
+ -Phadoop-aliyun-package,hadoop-aws-package,hadoop-azure-datalake-package\
+ -Phadoop-cos-package,hadoop-gcp-package,hadoop-tos-package
+
+The resulting tar file will be too large to be distributable through ASF infrastructure.
+
+The hadoop-gcp and hadoop-tos artifacts include their dependencies unless the distribution
+is built with -DskipShade.
+
 ----------------------------------------------------------------------------------
 Installing Hadoop
 

LICENSE-binary

@@ -214,11 +214,12 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/data
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
 
 ch.qos.reload4j:reload4j:1.2.22
+com.aliyun:aliyun-java-core:0.2.11-beta
 com.aliyun:aliyun-java-sdk-core:4.5.10
 com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
-com.aliyun:java-trace-api:0.2.11-beta.jar
+com.aliyun:java-trace-api:0.2.11-beta
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
@@ -274,7 +275,7 @@ com.google.j2objc:j2objc-annotations:3.0.0
 com.google.oauth-client:google-oauth-client:1.37.0
 com.huaweicloud:esdk-obs-java:3.20.4.2
 com.jamesmurty.utils:java-xmlbuilder-1.2.jar
-com.microsoft.azure:azure-storage:7.0.0
+com.microsoft.azure:azure-storage:7.0.1
 com.nimbusds:nimbus-jose-jwt:10.4
 com.squareup.okhttp3:okhttp:jar:3.14.2
 com.squareup.okio:okio:jar:1.17.2
@@ -556,7 +557,7 @@ com.microsoft.azure:azure-cosmosdb:2.4.5
 com.microsoft.azure:azure-cosmosdb-commons:2.4.5
 com.microsoft.azure:azure-cosmosdb-direct:2.4.5
 com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
-com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
+com.microsoft.azure:azure-data-lake-store-sdk:2.3.9
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
 org.bouncycastle:bcpkix-jdk18on:1.78.1

dev-support/bin/dist-layout-stitching

@@ -132,7 +132,7 @@ run copy "${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}"
 run copy "${ROOT}/hadoop-common-project/hadoop-registry/target/hadoop-registry-${VERSION}" .
 
 # cloud connectors go into common
-run copy "${ROOT}/hadoop-cloud-storage-project/hadoop-cloud-storage/target/hadoop-cloud-storage-${VERSION}" .
+run copy "${ROOT}/hadoop-cloud-storage-project/hadoop-cloud-storage-dist/target/hadoop-cloud-storage-dist-${VERSION}" .
 
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .

hadoop-cloud-storage-project/hadoop-cloud-storage-dist/pom.xml

@@ -37,15 +37,34 @@
   the Jars.
 
   By default, while hadoop-* artifacts are all included, dependencies
-  are omitted for nearly everything.
-  * keeps size down
-  * keeps CVE attack surface down
+  are omitted for all cloud connectors except hadoop-azure and
+  possibly hadoop-gcs and hadoop-tos modules.
+  For hadoop-aws the AWS SDK bundle.jar omitted, but everything else is included.
+
+   * This keeps binary release size below the limit of apache distributions
+   * Reduces download and size overhead in docker usage.
+   * Reduces the CVE attack surface
+   * Reduces the risk of classpath conflict.
+
   To produce a build with the specific desired dependencies, the build must be executed
-  with the relevant profile of ${module}-dependencies.
+  with the relevant profile of ${module}-package.
 
   For example, a build with the hadoop-aws and hadoop-azure-datalake dependencies,
-  run with
-      -Phadoop-aws-dependencies -Phadoop-azure-datalake-dependencies
+  build with -Dhadoop-aws-package -Dhadoop-azure-datalake-package
+
+  Available package profiles:
+    hadoop-aliyun-package
+    hadoop-aws-package
+    hadoop-azure-datalake-package
+    hadoop-cos-package
+    hadoop-gcp-package
+    hadoop-huaweicloud-package
+    hadoop-tos-package
+
+ To build a complete distribution then
+ mvn package -Pdist -DskipTests -Phadoop-aliyun-package,hadoop-aws-package,hadoop-azure-datalake-package\
+   -Phadoop-cos-package,hadoop-gcp-package,hadoop-tos-package
+
   -->
   <properties>
     <hadoop.component>cloud-storage</hadoop.component>
@@ -130,10 +149,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-gcp</artifactId>
       <scope>compile</scope>
-      <!--
-      Exclude transitive dependencies to prevent dependency convergence
-      problems. hadoop-gcp is a self-contained shaded jar.
-      -->
       <exclusions>
         <exclusion>
           <groupId>*</groupId>
@@ -202,12 +217,11 @@
       </build>
     </profile>
 
-
     <!-- Pull in aliyun -->
     <profile>
-      <id>hadoop-aliyun-dependencies</id>
+      <id>hadoop-aliyun-package</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>hadoop-aliyun-package</name></property>
       </activation>
       <dependencies>
         <dependency>
@@ -220,9 +234,9 @@
 
     <!-- Pull in the AWS SDK -->
     <profile>
-      <id>hadoop-aws-dependencies</id>
+      <id>hadoop-aws-package</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>hadoop-aws-package</name></property>
       </activation>
       <dependencies>
         <dependency>
@@ -233,41 +247,41 @@
       </dependencies>
     </profile>
 
-    <!-- Pull in all the cos -->
+    <!-- Pull in ADLS gen1 support -->
     <profile>
-      <id>hadoop-cos-dependencies</id>
+      <id>hadoop-azure-datalake-package</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>hadoop-azure-datalake-package</name></property>
       </activation>
       <dependencies>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-cos</artifactId>
+          <artifactId>hadoop-azure-datalake</artifactId>
           <scope>compile</scope>
         </dependency>
       </dependencies>
     </profile>
 
-    <!-- Pull in ADLS gen1 -->
+    <!-- Pull in all the hadoop-cos dependencies -->
     <profile>
-      <id>hadoop-azure-datalake-dependencies</id>
+      <id>hadoop-cos-package</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>hadoop-cos-package</name></property>
       </activation>
       <dependencies>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-azure-datalake</artifactId>
+          <artifactId>hadoop-cos</artifactId>
           <scope>compile</scope>
         </dependency>
       </dependencies>
     </profile>
 
     <!-- Pull in the huaweicloud dependencies -->
     <profile>
-      <id>hadoop-huaweicloud-dependencies</id>
+      <id>hadoop-huaweicloud-package</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>hadoop-huaweicloud-package</name></property>
       </activation>
       <dependencies>
         <dependency>
@@ -284,11 +298,26 @@
       </dependencies>
     </profile>
 
+    <!-- Pull in the gcp dependencies -->
+    <profile>
+      <id>hadoop-gcp-package</id>
+      <activation>
+        <property><name>hadoop-gcp-package</name></property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-gcp</artifactId>
+          <scope>compile</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+
     <!-- Pull in Volcano TOS -->
     <profile>
-      <id>hadoop-tos-dependencies</id>
+      <id>hadoop-tos-package</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>hadoop-tos-package</name></property>
       </activation>
       <dependencies>
         <dependency>

hadoop-cloud-storage-project/hadoop-tos/pom.xml

@@ -184,9 +184,9 @@
   <profiles>
     <!-- Generate a shaded artifact -->
     <profile>
-      <id>shade-tos</id>
+      <id>shade</id>
       <activation>
-        <activeByDefault>false</activeByDefault>
+        <property><name>!skipShade</name></property>
       </activation>
       <build>
         <plugins>

hadoop-project/pom.xml

@@ -2181,9 +2181,9 @@
         <version>2.4.4</version>
       </dependency>
       <dependency>
-      <groupId>com.google.cloud</groupId>
-      <artifactId>google-cloud-storage</artifactId>
-      <version>2.52.0</version>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-storage</artifactId>
+        <version>2.52.0</version>
     </dependency>
     </dependencies>
   </dependencyManagement>