Install AWS SDK on Iceberg image

trinodb · Apr 1, 2024 · 0e21e53 · 0e21e53
1 parent b14bf2f
commit 0e21e53
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/testing/spark3-iceberg/Dockerfile b/testing/spark3-iceberg/Dockerfile
@@ -32,6 +32,10 @@ RUN set -xeu; \
 
 WORKDIR ${SPARK_HOME}/jars
 
+# install AWS SDK so we can access S3; the version must match the hadoop-* jars which are part of SPARK distribution
+RUN wget -nv "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"
+RUN wget -nv "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.319/aws-java-sdk-bundle-1.12.319.jar"
+
 # install Iceberg
 RUN wget -nv "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_JAR_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_JAR_VERSION}-${ICEBERG_VERSION}.jar"