HDDS-11698. Use hadoop images from GitHub in CI (apache#7432)

swamirishi · Nov 17, 2024 · 3e798e6 · 3e798e6
1 parent 3e278b7
commit 3e798e6
Show file tree

Hide file tree

Showing 15 changed files with 33 additions and 23 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,6 +31,7 @@ env:
   # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
   TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  HADOOP_IMAGE: ghcr.io/apache/hadoop
   OZONE_IMAGE: ghcr.io/apache/ozone
   OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
   OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}

diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml
@@ -16,7 +16,7 @@
 
 services:
   rm:
-    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+    image: ${HADOOP_TEST_IMAGE}
     hostname: rm
     dns_search: .
     volumes:
@@ -35,7 +35,7 @@ services:
     profiles:
       - hadoop
   nm:
-    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+    image: ${HADOOP_TEST_IMAGE}
     hostname: nm
     dns_search: .
     volumes:
@@ -54,7 +54,7 @@ services:
     profiles:
       - hadoop
   jhs:
-    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+    image: ${HADOOP_TEST_IMAGE}
     container_name: jhs
     hostname: jhs
     dns_search: .

diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
@@ -21,14 +21,18 @@ if [[ ${SECURITY_ENABLED} == "true" ]]; then
 fi
 export COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml}":../common/${extra_compose_file}
 
-# need temp variables because maven filtering replaces only one item per line
-hadoop2_version="${hadoop2.version}"
-hadoop_version="${hadoop.version}"
-
-: ${HADOOP_TEST_VERSIONS:="apache/hadoop:${hadoop2_version} flokkr/hadoop:3.1.2 apache/hadoop:${hadoop_version}"}
+: ${HADOOP_IMAGE:="apache/hadoop"}
+: ${HADOOP_TEST_IMAGES:=""}
+
+if [[ -z "${HADOOP_TEST_IMAGES}" ]]; then
+  # hadoop2 and flokkr images are only available from Docker Hub
+  HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} apache/hadoop:${hadoop2.version}"
+  HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} flokkr/hadoop:3.1.2"
+  HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:${hadoop.version}"
+fi
 
 export HADOOP_MAJOR_VERSION=3
-export HADOOP_VERSION=unused # will be set for each test version below
+export HADOOP_TEST_IMAGE="${HADOOP_IMAGE}:${hadoop.version}"
 export OZONE_REPLICATION_FACTOR=3
 
 # shellcheck source=/dev/null
@@ -48,10 +52,10 @@ export OZONE_DIR=/opt/ozone
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
-for test_version in $HADOOP_TEST_VERSIONS; do
-  export HADOOP_IMAGE="${test_version%%:*}"
-  export HADOOP_VERSION="${test_version##*:}"
-  export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
+for HADOOP_TEST_IMAGE in $HADOOP_TEST_IMAGES; do
+  export HADOOP_TEST_IMAGE
+  hadoop_version="${HADOOP_TEST_IMAGE##*:}"
+  export HADOOP_MAJOR_VERSION=${hadoop_version%%.*}
 
   docker-compose --ansi never --profile hadoop up -d nm rm
 
@@ -62,10 +66,10 @@ for test_version in $HADOOP_TEST_VERSIONS; do
   fi
 
   for scheme in o3fs ofs; do
-    execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+    execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${hadoop_version}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
     # TODO secure MapReduce test is failing with 2.7 due to some token problem
     if [[ ${SECURITY_ENABLED} != "true" ]] || [[ ${HADOOP_MAJOR_VERSION} == "3" ]]; then
-      execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
+      execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${hadoop_version}-mapreduce-${scheme}" mapreduce.robot
     fi
   done
 

diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
@@ -16,7 +16,7 @@
 
 services:
   rm:
-    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+    image: ${HADOOP_TEST_IMAGE}
     hostname: rm
     volumes:
       - ../..:/opt/ozone
@@ -31,7 +31,7 @@ services:
     profiles:
       - hadoop
   nm:
-    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+    image: ${HADOOP_TEST_IMAGE}
     hostname: nm
     volumes:
       - ../..:/opt/ozone

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 HDDS_VERSION=${hdds.version}
+HADOOP_IMAGE=apache/hadoop
 HADOOP_VERSION=${hadoop.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
@@ -27,7 +27,7 @@ services:
       ozone_net:
         ipv4_address: 172.25.0.100
   kms:
-    image: apache/hadoop:${HADOOP_VERSION}
+    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
     dns_search: .
     ports:
       - 9600:9600

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
@@ -16,6 +16,7 @@
 
 HDDS_VERSION=${hdds.version}
 HADOOP_IMAGE=apache/hadoop
+HADOOP_VERSION=${hadoop.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
       - ../_keytabs:/etc/security/keytabs
     command: ["krb5kdc","-n"]
   kms:
-    image: apache/hadoop:3
+    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
     dns_search: .
     ports:
       - 9600:9600

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 HDDS_VERSION=${hdds.version}
+HADOOP_IMAGE=apache/hadoop
 HADOOP_VERSION=${hadoop.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
       - ../_keytabs:/etc/security/keytabs
     command: ["krb5kdc","-n"]
   kms:
-    image: apache/hadoop:${HADOOP_VERSION}
+    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
     hostname: kms
     dns_search: .
     ports:

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+HADOOP_IMAGE=apache/hadoop
 HADOOP_VERSION=${hadoop.version}
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
@@ -83,7 +83,7 @@ services:
       - docker-config
     environment:
       HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
-    image: apache/hadoop:${HADOOP_VERSION}
+    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
     networks:
       net:
         ipv4_address: 10.9.0.3

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env
@@ -17,6 +17,7 @@
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
+HADOOP_IMAGE=apache/hadoop
 HADOOP_VERSION=${hadoop.version}
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
 OZONE_IMAGE=${docker.ozone.image}

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml
@@ -36,7 +36,7 @@ services:
       - ../_keytabs:/etc/security/keytabs
     command: [ "krb5kdc","-n" ]
   kms:
-    image: apache/hadoop:${HADOOP_VERSION}
+    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
     hostname: kms
     dns_search: .
     ports:

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
@@ -36,7 +36,7 @@ services:
       - ../_keytabs:/etc/security/keytabs
     command: [ "krb5kdc","-n" ]
   kms:
-    image: apache/hadoop:${HADOOP_VERSION}
+    image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
     hostname: kms
     dns_search: .
     ports: