diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 42dc19ad266..09f47dd075f 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -102,7 +102,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") - environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.0") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 9baa2abafac..65a36cd6b37 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -165,7 +165,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") - environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.1") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/dev/docker/kerberos-hive/start.sh b/dev/docker/kerberos-hive/start.sh index e937a12309b..0ff1bda13ec 100644 --- a/dev/docker/kerberos-hive/start.sh +++ b/dev/docker/kerberos-hive/start.sh @@ -7,8 +7,10 @@ # start ssh HOSTNAME=`hostname` service ssh start -ssh-keyscan localhost > /root/.ssh/known_hosts +ssh-keyscan ${HOSTNAME} >> /root/.ssh/known_hosts +ssh-keyscan localhost >> /root/.ssh/known_hosts ssh-keyscan 0.0.0.0 >> /root/.ssh/known_hosts +ssh-keyscan 127.0.0.1 >> /root/.ssh/known_hosts # init the Kerberos database echo -e "${PASS}\n${PASS}" | kdb5_util create -s @@ -65,6 +67,37 @@ ${HADOOP_HOME}/sbin/hadoop-daemon.sh start namenode echo "Starting DataNode..." ${HADOOP_HOME}/sbin/start-secure-dns.sh +sleep 5 + +# Check if the DataNode is running +ps -ef | grep DataNode | grep -v "color=auto" +if [[ $? -ne 0 ]]; then + echo "DataNode failed to start, please check the logs" + ehco "HDFS DataNode log start----------------------------" + cat ${HADOOP_HOME}/bin/logs/hadoop-root-datanode-*.log + exit 1 +fi + +retry_times=0 +ready=0 +while [[ ${retry_times} -lt 10 ]]; do + hdfs_ready=$(hdfs dfsadmin -report | grep "Live datanodes" | awk '{print $3}') + if [[ ${hdfs_ready} == "(1):" ]]; then + echo "HDFS is ready, retry_times = ${retry_times}" + let "ready=0" + break + fi + sleep 10 + retry_times=$((retry_times+1)) +done + +if [[ ${ready} -ne 0 ]]; then + echo "HDFS is not ready" + ehco "HDFS DataNode log start---------------------------" + cat ${HADOOP_HOME}/bin/logs/hadoop-root-datanode-*.log + exit 1 +fi + # start mysql and create databases/users for hive chown -R mysql:mysql /var/lib/mysql diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index 3995a737afc..7f4d50c7ac8 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -89,6 +89,9 @@ You can use these kinds of Docker images to facilitate integration testing of al You can use this kind of image to test the catalog of Apache Hive with kerberos enable Changelog +- gravitino-ci-kerberos-hive:0.1.2 + - Add `${HOSTNAME} >> /root/.ssh/known_hosts` to the startup script. + - Add check for the status of DataNode, if the DataNode is not running or ready within 100s, the container will exit. - gravitino-ci-kerberos-hive:0.1.1 - Add a principal for Gravitino web server named 'HTTP/localhost@HADOOPKRB'.