Optimize the HMS Docker image and enable S3 support

Author: deniskuzZ

metastore/src/java/org/apache/hadoop/hive/metastore/HiveProtoEventsCleanerTask.java

@@ -26,12 +26,12 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.util.SystemClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
@@ -48,7 +48,6 @@ public class HiveProtoEventsCleanerTask implements MetastoreTaskThread {
   private Configuration conf;
   private long ttl;
   private static String expiredDatePtn = null;
-  private static final SystemClock clock = SystemClock.getInstance();
 
   @Override
   public void setConf(Configuration conf) {
@@ -95,9 +94,11 @@ public void run() {
    * Compute the expired date partition, using the underlying clock in UTC time.
    */
   private static void computeExpiredDatePtn(long ttl) {
+    LocalDate expiredDate = LocalDate.ofInstant(
+        Instant.now().minusMillis(ttl),
+        ZoneOffset.UTC
+    );
     // Use UTC date to ensure reader date is same on all timezones.
-    LocalDate expiredDate
-            = LocalDateTime.ofEpochSecond((clock.getTime() - ttl) / 1000, 0, ZoneOffset.UTC).toLocalDate();
     expiredDatePtn = "date=" + DateTimeFormatter.ISO_LOCAL_DATE.format(expiredDate);
   }
 

standalone-metastore/packaging/src/docker/Dockerfile

@@ -42,22 +42,46 @@ RUN echo ${BUILD_ENV}
 ARG HADOOP_VERSION
 ARG HIVE_VERSION
 
-RUN tar -xzvf /opt/hadoop-$HADOOP_VERSION.tar.gz -C /opt/ && \
-    rm -rf /opt/hadoop-$HADOOP_VERSION/share/doc/* && \
+RUN apt-get update && \
+    apt-get install -y wget
+
+RUN tar -xzv \
+      --exclude="hadoop-$HADOOP_VERSION/include" \
+      --exclude="hadoop-$HADOOP_VERSION/lib/native" \
+      --exclude="hadoop-$HADOOP_VERSION/share/doc" \
+      --exclude="hadoop-$HADOOP_VERSION/share/hadoop/client" \
+      --exclude="hadoop-$HADOOP_VERSION/share/hadoop/tools" \
+      --exclude="hadoop-$HADOOP_VERSION/share/hadoop/yarn/*" \
+      --exclude="*/jdiff" \
+      --exclude="*/sources" \
+      --exclude="*tests.jar" \
+      --exclude="*/webapps" \
+      -f /opt/hadoop-$HADOOP_VERSION.tar.gz \
+      -C /opt/ && \
+    \
+    find /opt/hadoop-$HADOOP_VERSION/share/hadoop/common/lib \
+      \( -name "jetty-*.jar" -o -name "zookeeper-*.jar" -o -name "netty-*.jar" \) \
+      -delete && \
+    # Extract hadoop jars only
+    tar -xzv  \
+      -f /opt/hadoop-$HADOOP_VERSION.tar.gz \
+      -C /opt/ \
+      --wildcards "hadoop-$HADOOP_VERSION/share/hadoop/tools/lib/hadoop-*.jar" && \
+    # INSTALL HIVE
     tar -xzvf /opt/hive-standalone-metastore-$HIVE_VERSION-bin.tar.gz -C /opt/
 
 FROM eclipse-temurin:21.0.3_9-jre-ubi9-minimal AS run
 
+ARG UID=1000
 ARG HADOOP_VERSION
 ARG HIVE_VERSION
-COPY --from=env /opt/hadoop-$HADOOP_VERSION /opt/hadoop
-COPY --from=env /opt/apache-hive-metastore-$HIVE_VERSION-bin /opt/hive
 
 # Install dependencies
 RUN set -ex; \
     microdnf update -y; \
     microdnf -y install procps; \
-    rm -rf /var/lib/apt/lists/*
+    microdnf clean all; \
+    useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive
 
 # Set necessary environment variables.
 ENV HADOOP_HOME=/opt/hadoop \
@@ -66,20 +90,18 @@ ENV HADOOP_HOME=/opt/hadoop \
 
 ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH
 
-COPY entrypoint.sh /
-COPY conf $HIVE_HOME/conf
-RUN chmod +x /entrypoint.sh
+COPY --from=env --chown=hive /opt/hadoop-$HADOOP_VERSION $HADOOP_HOME
+COPY --from=env --chown=hive /opt/apache-hive-metastore-$HIVE_VERSION-bin $HIVE_HOME
 
+COPY --chown=hive entrypoint.sh /
+COPY --chown=hive conf $HIVE_HOME/conf
 
-ARG UID=1000
-RUN useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive && \
-    chown hive /opt/hive && \
-    chown hive /opt/hadoop && \
-    chown hive /opt/hive/conf && \
-    mkdir -p /opt/hive/data/warehouse && \
-    chown hive /opt/hive/data/warehouse
+RUN chmod +x /entrypoint.sh && \
+    mkdir -p $HIVE_HOME/data/warehouse && \
+    chown hive $HIVE_HOME/data/warehouse
 
 USER hive
-WORKDIR /opt/hive
+WORKDIR $HIVE_HOME
 EXPOSE 9001 9083
+
 ENTRYPOINT ["sh", "-c", "/entrypoint.sh"]

standalone-metastore/packaging/src/docker/README.md

@@ -131,6 +131,13 @@ export POSTGRES_LOCAL_PATH=`mvn help:evaluate -Dexpression=settings.localReposit
 If you don't install maven or have problem in resolving the postgres driver, you can always download this jar yourself,
 change the `POSTGRES_LOCAL_PATH` to the path of the downloaded jar.
 
+- Metastore with S3 support
+
+Download aws-java-sdk-bundle-xxx.jar and place it under the jars directory:
+wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.770/aws-java-sdk-bundle-1.12.770.jar -P jars/
+
+Add the `fs.s3a.access.key` and `fs.s3a.secret.key` properties in `metastore-site.xml` under the conf directory.
+
 Then,
 ```shell
 docker compose up -d

standalone-metastore/packaging/src/docker/conf/metastore-site.xml

@@ -32,4 +32,15 @@
     <name>metastore.catalog.servlet.auth</name>
     <value>none</value>
   </property>
+
+  <!--
+    <property>
+      <name>fs.s3a.access.key</name>
+      <value>YOUR_AWS_ACCESS_KEY</value>
+    </property>
+    <property>
+      <name>fs.s3a.secret.key</name>
+      <value>YOUR_AWS_SECRET_KEY</value>
+    </property>
+    -->
 </configuration>

standalone-metastore/packaging/src/docker/docker-compose.yml

@@ -54,6 +54,8 @@ services:
         - type: bind
           source: ${POSTGRES_LOCAL_PATH}
           target: /opt/hive/lib/postgres.jar
+        # Mount local jars to a temporary staging area (Read-Only)
+        - ./jars:/tmp/ext-jars:ro
     networks:
       - hive
 

standalone-metastore/packaging/src/docker/entrypoint.sh

@@ -19,16 +19,37 @@
 
 set -x
 
+# =========================================================================
+# DYNAMIC JAR LOADER (AWS/S3 Support)
+# =========================================================================
+STAGING_DIR="/tmp/ext-jars"
+
+TOOLS_LIB="${HADOOP_HOME}/share/hadoop/tools/lib"
+COMMON_LIB="${HADOOP_HOME}/share/hadoop/common/lib"
+
+# Checks if /tmp/ext-jars is mounted (via Docker volume).
+if [ -d "$STAGING_DIR" ]; then
+  # Check for aws-java-sdk-bundle (Wildcard handles versions)
+  if ls "$STAGING_DIR"/aws-java-sdk-bundle-*.jar 1> /dev/null 2>&1; then
+    echo "--> Installing AWS SDK Bundle..."
+    cp "$STAGING_DIR"/aws-java-sdk-bundle-*.jar "$COMMON_LIB/"
+    echo "--> activating hadoop-aws from tools..."
+    cp "$TOOLS_LIB"/hadoop-aws-*.jar "$COMMON_LIB/"
+  fi
+fi
+
 : "${DB_DRIVER:=derby}"
 
 SKIP_SCHEMA_INIT="${IS_RESUME:-false}"
 [[ $VERBOSE = "true" ]] && VERBOSE_MODE="--verbose" || VERBOSE_MODE=""
 
 function initialize_hive {
   COMMAND="-initOrUpgradeSchema"
+  # Check Hive version. If < 4.0.0, use older initSchema command
   if [ "$(echo "$HIVE_VER" | cut -d '.' -f1)" -lt "4" ]; then
      COMMAND="-${SCHEMA_COMMAND:-initSchema}"
   fi
+
   "$HIVE_HOME/bin/schematool" -dbType "$DB_DRIVER" "$COMMAND" "$VERBOSE_MODE"
   if [ $? -eq 0 ]; then
     echo "Initialized Hive Metastore Server schema successfully.."
@@ -39,17 +60,21 @@ function initialize_hive {
 }
 
 export HIVE_CONF_DIR=$HIVE_HOME/conf
+
 if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then
   find "${HIVE_CUSTOM_CONF_DIR}" -type f -exec \
     ln -sfn {} "${HIVE_CONF_DIR}"/ \;
   export HADOOP_CONF_DIR=$HIVE_CONF_DIR
 fi
 
 export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx1G $SERVICE_OPTS"
+
 if [[ "${SKIP_SCHEMA_INIT}" == "false" ]]; then
   # handles schema initialization
   initialize_hive
 fi
 
 export METASTORE_PORT=${METASTORE_PORT:-9083}
-exec "$HIVE_HOME/bin/start-metastore" 
+
+echo "Starting Hive Metastore on port $METASTORE_PORT..."
+exec "$HIVE_HOME/bin/start-metastore"