diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh index 571059be6fd0..8bbe17fb7e60 100755 --- a/dev/make-distribution.sh +++ b/dev/make-distribution.sh @@ -183,6 +183,10 @@ echo "Build flags: $@" >> "$DISTDIR/RELEASE" # Copy jars cp "$SPARK_HOME"/assembly/target/scala*/jars/* "$DISTDIR/jars/" +if [ -f "$SPARK_HOME"/hadoop-cloud/target/hadoop-aws-*-shaded.jar ]; then + cp "$SPARK_HOME"/hadoop-cloud/target/hadoop-aws-*-shaded.jar "$DISTDIR/jars" +fi + # Only create the yarn directory if the yarn artifacts were built. if [ -f "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar ]; then mkdir "$DISTDIR/yarn" diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index a5642a5a68fe..9b735a9da68f 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -34,6 +34,7 @@ hadoop-cloud + ${hadoop.deps.scope} @@ -75,7 +76,7 @@ org.apache.hadoop hadoop-aws ${hadoop.version} - ${hadoop.deps.scope} + ${hadoop-shaded.deps.scope} org.apache.hadoop @@ -211,10 +212,52 @@ src/hadoop-3/main/scala src/hadoop-3/test/scala + + provided + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + true + ${project.build.directory}/hadoop-aws-${hadoop.version}-shaded.jar + + + org.apache.hadoop:hadoop-aws + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + com.google.common + org.apache.hadoop.shaded.com.google.common + + com.google.common.** + + + + + org.codehaus.mojo build-helper-maven-plugin @@ -274,6 +317,10 @@ com.google.guava guava + + org.apache.hadoop + hadoop-aws +