HDDS-11686. Use ozone image from GitHub in CI (apache#7425)

ptlrs · Nov 13, 2024 · 10cac80 · 10cac80
1 parent aa6da3e
commit 10cac80
Show file tree

Hide file tree

Showing 13 changed files with 24 additions and 15 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,6 +31,7 @@ env:
   # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
   TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  OZONE_IMAGE: ghcr.io/apache/ozone
   OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
   OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
 jobs:

diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
@@ -28,6 +28,8 @@
   <properties>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
+    <docker.ozone.image>apache/ozone</docker.ozone.image>
+    <docker.ozone.image.flavor>-rocky</docker.ozone.image.flavor> <!-- suffix appended to Ozone version to get Docker image version -->
     <docker.ozone-runner.version>20241108-jdk17-1</docker.ozone-runner.version>
     <docker.ozone-testkr5b.image>ghcr.io/apache/ozone-testkrb5:20241112-1</docker.ozone-testkr5b.image>
     <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->

diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -517,9 +517,13 @@ fix_data_dir_permissions() {
 ## @param `ozone` image version
 prepare_for_binary_image() {
   local v=$1
+  local default_image="${docker.ozone.image}" # set at build-time from Maven property
+  local default_flavor="${docker.ozone.image.flavor}" # set at build-time from Maven property
+  local image="${OZONE_IMAGE:-${default_image}}" # may be specified by user running the test
+  local flavor="${OZONE_IMAGE_FLAVOR:-${default_flavor}}" # may be specified by user running the test
 
   export OZONE_DIR=/opt/ozone
-  export OZONE_IMAGE="apache/ozone:${v}"
+  export OZONE_TEST_IMAGE="${image}:${v}${flavor}"
 }
 
 ## @description Define variables required for using `ozone-runner` docker image
@@ -539,7 +543,7 @@ get_runner_image_spec() {
 ## @param `ozone-runner` image version (optional)
 prepare_for_runner_image() {
   export OZONE_DIR=/opt/hadoop
-  export OZONE_IMAGE="$(get_runner_image_spec "$@")"
+  export OZONE_TEST_IMAGE="$(get_runner_image_spec "$@")"
 }
 
 ## @description Executing the Ozone Debug CLI related robot tests

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
@@ -18,7 +18,7 @@ HADOOP_VERSION=${hadoop.version}
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
+OZONE_TEST_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
 OZONE_DIR=/opt/hadoop
 OZONE_VOLUME=./data

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
@@ -20,7 +20,7 @@ x-common-config:
   env_file:
     - docker-config
     - ../../../common/security.conf
-  image: ${OZONE_IMAGE}
+  image: ${OZONE_TEST_IMAGE}
   dns_search: .
 
 x-environment:

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/.env
@@ -17,6 +17,6 @@
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
+OZONE_TEST_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
 OZONE_DIR=/opt/hadoop
 OZONE_VOLUME=./data
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-compose.yaml
@@ -19,7 +19,7 @@ x-common-config:
   &common-config
   env_file:
     - docker-config
-  image: ${OZONE_IMAGE}
+  image: ${OZONE_TEST_IMAGE}
 
 x-environment:
   &environment

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/.env
@@ -17,7 +17,7 @@
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
+OZONE_TEST_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
 OZONE_DIR=/opt/hadoop
 OZONE_VOLUME=./data
 OM_SERVICE_ID=omservice

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-compose.yaml
@@ -19,7 +19,7 @@ x-common-config:
   &common-config
   env_file:
     - docker-config
-  image: ${OZONE_IMAGE}
+  image: ${OZONE_TEST_IMAGE}
 
 x-environment:
   &environment

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
@@ -56,7 +56,7 @@ prepare_for_image() {
   if [[ "$image_version" = "$OZONE_CURRENT_VERSION" ]]; then
       prepare_for_runner_image
   else
-      prepare_for_binary_image "${image_version}-rocky"
+      prepare_for_binary_image "${image_version}"
   fi
 }
 

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env
@@ -19,3 +19,5 @@ OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 HADOOP_VERSION=${hadoop.version}
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
+OZONE_IMAGE=${docker.ozone.image}
+OZONE_IMAGE_FLAVOR="${docker.ozone.image.flavor}"
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
@@ -16,7 +16,7 @@
 
 services:
   old_client_1_0_0:
-    image: apache/ozone:1.0.0-rocky
+    image: ${OZONE_IMAGE}:1.0.0${OZONE_IMAGE_FLAVOR}
     env_file:
       - docker-config
     volumes:
@@ -25,7 +25,7 @@ services:
       - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_1_0:
-    image: apache/ozone:1.1.0-rocky
+    image: ${OZONE_IMAGE}:1.1.0${OZONE_IMAGE_FLAVOR}
     env_file:
       - docker-config
     volumes:
@@ -34,7 +34,7 @@ services:
       - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_2_1:
-    image: apache/ozone:1.2.1-rocky
+    image: ${OZONE_IMAGE}:1.2.1${OZONE_IMAGE_FLAVOR}
     env_file:
       - docker-config
     volumes:
@@ -43,7 +43,7 @@ services:
       - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_3_0:
-    image: apache/ozone:1.3.0-rocky
+    image: ${OZONE_IMAGE}:1.3.0${OZONE_IMAGE_FLAVOR}
     env_file:
       - docker-config
     volumes:
@@ -52,7 +52,7 @@ services:
       - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_4_0:
-    image: apache/ozone:1.4.0-rocky
+    image: ${OZONE_IMAGE}:1.4.0${OZONE_IMAGE_FLAVOR}
     env_file:
       - docker-config
     volumes:

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
@@ -17,7 +17,7 @@
 # reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields)
 x-old-config:
   &old-config
-  image: apache/ozone:${OZONE_VERSION}-rocky
+  image: ${OZONE_IMAGE}:${OZONE_VERSION}${OZONE_IMAGE_FLAVOR}
   dns_search: .
   env_file:
     - docker-config