fixed hive docker

Author: deniskuzZ

packaging/src/docker/Dockerfile

@@ -59,18 +59,17 @@ RUN tar -xzvf /opt/hadoop-$HADOOP_VERSION.tar.gz -C /opt/ && \
 
 FROM eclipse-temurin:21.0.3_9-jre-ubi9-minimal AS run
 
+ARG UID=1000
 ARG HADOOP_VERSION
 ARG HIVE_VERSION
 ARG TEZ_VERSION
-COPY --from=env /opt/hadoop-$HADOOP_VERSION /opt/hadoop
-COPY --from=env /opt/apache-hive-$HIVE_VERSION-bin /opt/hive
-COPY --from=env /opt/apache-tez-$TEZ_VERSION-bin /opt/tez
 
 # Install dependencies
 RUN set -ex; \
     microdnf update -y; \
     microdnf -y install procps; \
-    rm -rf /var/lib/apt/lists/*
+    microdnf clean all; \
+    useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive
 
 # Set necessary environment variables.
 ENV HADOOP_HOME=/opt/hadoop \
@@ -80,23 +79,21 @@ ENV HADOOP_HOME=/opt/hadoop \
 
 ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH
 
-COPY entrypoint.sh /
-COPY conf $HIVE_HOME/conf
-RUN chmod +x /entrypoint.sh
+COPY --from=env --chown=hive /opt/hadoop-$HADOOP_VERSION $HADOOP_HOME
+COPY --from=env --chown=hive /opt/apache-hive-$HIVE_VERSION-bin $HIVE_HOME
+COPY --from=env --chown=hive /opt/apache-tez-$TEZ_VERSION-bin $TEZ_HOME
 
+COPY --chown=hive entrypoint.sh /
+COPY --chown=hive conf $HIVE_HOME/conf
 
-ARG UID=1000
-RUN useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive && \
-    chown hive /opt/tez && \
-    chown hive /opt/hive && \
-    chown hive /opt/hadoop && \
-    chown hive /opt/hive/conf && \
-    mkdir -p /opt/hive/data/warehouse && \
-    chown hive /opt/hive/data/warehouse && \
+RUN chmod +x /entrypoint.sh && \
+    mkdir -p $HIVE_HOME/data/warehouse && \
+    chown hive $HIVE_HOME/data/warehouse && \
     mkdir -p /home/hive/.beeline && \
     chown hive /home/hive/.beeline
 
 USER hive
 WORKDIR /opt/hive
 EXPOSE 10000 10002 9083
+
 ENTRYPOINT ["sh", "-c", "/entrypoint.sh"]

packaging/src/docker/conf/hive-site.xml

@@ -56,10 +56,6 @@
         <name>tez.local.mode</name>
         <value>true</value>
     </property>
-    <property>
-        <name>hive.metastore.warehouse.dir</name>
-        <value>/opt/hive/data/warehouse</value>
-    </property>
     <property>
         <name>hive.metastore.event.db.notification.api.auth</name>
         <value>false</value>

packaging/src/docker/docker-compose.yml

@@ -33,13 +33,18 @@ services:
       - hive
 
   metastore:
-    image: apache/hive:${HIVE_VERSION}
+    image: apache/hive:standalone-metastore-${HIVE_VERSION}
     depends_on:
       - postgres
     restart: unless-stopped
     container_name: metastore
     hostname: metastore
     environment:
+      HIVE_OPTS: >
+        -hiveconf fs.s3a.access.key=${AWS_ACCESS_KEY_ID}
+        -hiveconf fs.s3a.secret.key=${AWS_SECRET_ACCESS_KEY}
+        -hiveconf fs.s3a.endpoint=${FS_S3A_ENDPOINT}
+      HADOOP_CLASSPATH: /opt/hadoop/share/hadoop/tools/lib/*
       DB_DRIVER: postgres
       SERVICE_NAME: 'metastore'
       SERVICE_OPTS: '-Xmx1G -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver
@@ -53,6 +58,8 @@ services:
         - type: bind
           source: ${POSTGRES_LOCAL_PATH}
           target: /opt/hive/lib/postgres.jar
+        # Mount local jars to a temporary staging area (Read-Only)
+        - ./jars:/tmp/ext-jars:ro
     networks:
       - hive
 
@@ -63,6 +70,9 @@ services:
     restart: unless-stopped
     container_name: hiveserver2
     environment:
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      HADOOP_CLASSPATH: /opt/hadoop/share/hadoop/tools/lib/*
       HIVE_SERVER2_THRIFT_PORT: 10000
       SERVICE_OPTS: '-Xmx1G -Dhive.metastore.uris=thrift://metastore:9083'
       IS_RESUME: 'true'
@@ -72,6 +82,8 @@ services:
       - '10002:10002'
     volumes:
       - warehouse:/opt/hive/data/warehouse
+      # Mount local jars to a temporary staging area (Read-Only)
+      - ./jars:/tmp/ext-jars:ro
     networks:
       - hive
 

packaging/src/docker/entrypoint.sh

@@ -19,6 +19,21 @@
 
 set -x
 
+# =========================================================================
+# DYNAMIC JAR LOADER (AWS/S3 Support)
+# =========================================================================
+STAGING_DIR="/tmp/ext-jars"
+
+# Checks if /tmp/ext-jars is mounted (via Docker volume).
+if [ -d "$STAGING_DIR" ]; then
+  if ls "$STAGING_DIR"/*.jar 1> /dev/null 2>&1; then
+    echo "--> Copying custom jars from volume to Hive..."
+    cp -vf "$STAGING_DIR"/*.jar "${HIVE_HOME}/lib/"
+  else
+    echo "--> Volume mounted at $STAGING_DIR, but no jars found."
+  fi
+fi
+
 : "${DB_DRIVER:=derby}"
 
 SKIP_SCHEMA_INIT="${IS_RESUME:-false}"

standalone-metastore/packaging/src/docker/README.md

@@ -139,10 +139,10 @@ Note: Hadoop 3.4.1 requires both AWS SDK V1 and V2 for its S3A connector:
 wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.770/aws-java-sdk-bundle-1.12.770.jar -P jars/
 wget https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.26.19/bundle-2.26.19.jar -P jars/
 
-Then add the following configs to the `metastore-site.xml` under the conf directory:
-- fs.s3a.endpoint
-- fs.s3a.access.key
-- fs.s3a.secret.key
+Then add the following ENV variables:
+- AWS_ACCESS_KEY_ID
+- AWS_SECRET_ACCESS_KEY
+- FS_S3A_ENDPOINT
 
 Then,
 ```shell

standalone-metastore/packaging/src/docker/conf/metastore-site.xml

@@ -32,19 +32,4 @@
     <name>metastore.catalog.servlet.auth</name>
     <value>none</value>
   </property>
-
-  <!--
-    <property>
-      <name>fs.s3a.endpoint</name>
-      <value>s3.us-west-2.amazonaws.com</value>
-    </property>
-    <property>
-      <name>fs.s3a.access.key</name>
-      <value>YOUR_AWS_ACCESS_KEY</value>
-    </property>
-    <property>
-      <name>fs.s3a.secret.key</name>
-      <value>YOUR_AWS_SECRET_KEY</value>
-    </property>
-    -->
 </configuration>

standalone-metastore/packaging/src/docker/docker-compose.yml

@@ -40,6 +40,11 @@ services:
     container_name: metastore
     hostname: metastore
     environment:
+      HIVE_OPTS: >
+        -hiveconf fs.s3a.access.key=${AWS_ACCESS_KEY_ID}
+        -hiveconf fs.s3a.secret.key=${AWS_SECRET_ACCESS_KEY}
+        -hiveconf fs.s3a.endpoint=${FS_S3A_ENDPOINT}
+      HADOOP_CLASSPATH: /opt/hadoop/share/hadoop/tools/lib/*
       DB_DRIVER: postgres
       SERVICE_NAME: 'metastore'
       SERVICE_OPTS: '-Xmx1G -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver

standalone-metastore/packaging/src/docker/entrypoint.sh

@@ -24,14 +24,11 @@ set -x
 # =========================================================================
 STAGING_DIR="/tmp/ext-jars"
 
-TOOLS_LIB="${HADOOP_HOME}/share/hadoop/tools/lib"
-HIVE_LIB="${HIVE_HOME}/lib/"
-
 # Checks if /tmp/ext-jars is mounted (via Docker volume).
 if [ -d "$STAGING_DIR" ]; then
   if ls "$STAGING_DIR"/*.jar 1> /dev/null 2>&1; then
     echo "--> Copying custom jars from volume to Hive..."
-    cp -vf "$STAGING_DIR"/*.jar "$HIVE_LIB/"
+    cp -vf "$STAGING_DIR"/*.jar "${HIVE_HOME}/lib/"
   else
     echo "--> Volume mounted at $STAGING_DIR, but no jars found."
   fi
@@ -74,6 +71,4 @@ if [[ "${SKIP_SCHEMA_INIT}" == "false" ]]; then
 fi
 
 export METASTORE_PORT=${METASTORE_PORT:-9083}
-
-echo "Starting Hive Metastore on port $METASTORE_PORT..."
 exec "$HIVE_HOME/bin/start-metastore"