diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bd913b53b8..bf180b15001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ env: # Minimum required Java version for running Ozone is defined in pom.xml (javac.version). TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + HADOOP_IMAGE: ghcr.io/apache/hadoop OZONE_IMAGE: ghcr.io/apache/ozone OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }} diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml index 8600659b786..ee97525fbb0 100644 --- a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml +++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml @@ -16,7 +16,7 @@ services: rm: - image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + image: ${HADOOP_TEST_IMAGE} hostname: rm dns_search: . volumes: @@ -35,7 +35,7 @@ services: profiles: - hadoop nm: - image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + image: ${HADOOP_TEST_IMAGE} hostname: nm dns_search: . volumes: @@ -54,7 +54,7 @@ services: profiles: - hadoop jhs: - image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + image: ${HADOOP_TEST_IMAGE} container_name: jhs hostname: jhs dns_search: . diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh index 7bf56074437..c16f6fe22f5 100755 --- a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh @@ -21,14 +21,18 @@ if [[ ${SECURITY_ENABLED} == "true" ]]; then fi export COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml}":../common/${extra_compose_file} -# need temp variables because maven filtering replaces only one item per line -hadoop2_version="${hadoop2.version}" -hadoop_version="${hadoop.version}" - -: ${HADOOP_TEST_VERSIONS:="apache/hadoop:${hadoop2_version} flokkr/hadoop:3.1.2 apache/hadoop:${hadoop_version}"} +: ${HADOOP_IMAGE:="apache/hadoop"} +: ${HADOOP_TEST_IMAGES:=""} + +if [[ -z "${HADOOP_TEST_IMAGES}" ]]; then + # hadoop2 and flokkr images are only available from Docker Hub + HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} apache/hadoop:${hadoop2.version}" + HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} flokkr/hadoop:3.1.2" + HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:${hadoop.version}" +fi export HADOOP_MAJOR_VERSION=3 -export HADOOP_VERSION=unused # will be set for each test version below +export HADOOP_TEST_IMAGE="${HADOOP_IMAGE}:${hadoop.version}" export OZONE_REPLICATION_FACTOR=3 # shellcheck source=/dev/null @@ -48,10 +52,10 @@ export OZONE_DIR=/opt/ozone # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" -for test_version in $HADOOP_TEST_VERSIONS; do - export HADOOP_IMAGE="${test_version%%:*}" - export HADOOP_VERSION="${test_version##*:}" - export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*} +for HADOOP_TEST_IMAGE in $HADOOP_TEST_IMAGES; do + export HADOOP_TEST_IMAGE + hadoop_version="${HADOOP_TEST_IMAGE##*:}" + export HADOOP_MAJOR_VERSION=${hadoop_version%%.*} docker-compose --ansi never --profile hadoop up -d nm rm @@ -62,10 +66,10 @@ for test_version in $HADOOP_TEST_VERSIONS; do fi for scheme in o3fs ofs; do - execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot + execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${hadoop_version}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot # TODO secure MapReduce test is failing with 2.7 due to some token problem if [[ ${SECURITY_ENABLED} != "true" ]] || [[ ${HADOOP_MAJOR_VERSION} == "3" ]]; then - execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot + execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoop-${hadoop_version}-mapreduce-${scheme}" mapreduce.robot fi done diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml index 4fb56e5aa98..c5899eb6b4f 100644 --- a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml +++ b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml @@ -16,7 +16,7 @@ services: rm: - image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + image: ${HADOOP_TEST_IMAGE} hostname: rm volumes: - ../..:/opt/ozone @@ -31,7 +31,7 @@ services: profiles: - hadoop nm: - image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + image: ${HADOOP_TEST_IMAGE} hostname: nm volumes: - ../..:/opt/ozone diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env index 5f3e96ed617..c3a2c5329aa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env @@ -15,6 +15,7 @@ # limitations under the License. HDDS_VERSION=${hdds.version} +HADOOP_IMAGE=apache/hadoop HADOOP_VERSION=${hadoop.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml index d55d5e0e2e8..354f718ade7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml @@ -27,7 +27,7 @@ services: ozone_net: ipv4_address: 172.25.0.100 kms: - image: apache/hadoop:${HADOOP_VERSION} + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} dns_search: . ports: - 9600:9600 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env index cb5ab1080ac..c3a2c5329aa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env @@ -16,6 +16,7 @@ HDDS_VERSION=${hdds.version} HADOOP_IMAGE=apache/hadoop +HADOOP_VERSION=${hadoop.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml index 625aac77723..f0b094447d0 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml @@ -24,7 +24,7 @@ services: - ../_keytabs:/etc/security/keytabs command: ["krb5kdc","-n"] kms: - image: apache/hadoop:3 + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} dns_search: . ports: - 9600:9600 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env index 5f3e96ed617..c3a2c5329aa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env @@ -15,6 +15,7 @@ # limitations under the License. HDDS_VERSION=${hdds.version} +HADOOP_IMAGE=apache/hadoop HADOOP_VERSION=${hadoop.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml index 026dfa1edc3..f0b4aee1b35 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml @@ -24,7 +24,7 @@ services: - ../_keytabs:/etc/security/keytabs command: ["krb5kdc","-n"] kms: - image: apache/hadoop:${HADOOP_VERSION} + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: kms dns_search: . ports: diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env index dee41e9e66b..2625c4fbe90 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +HADOOP_IMAGE=apache/hadoop HADOOP_VERSION=${hadoop.version} HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index 0d0af747ee1..86ebcc0c2b0 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -83,7 +83,7 @@ services: - docker-config environment: HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop - image: apache/hadoop:${HADOOP_VERSION} + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} networks: net: ipv4_address: 10.9.0.3 diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env index 6ace68e8938..11979d34326 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/.env +++ b/hadoop-ozone/dist/src/main/compose/xcompat/.env @@ -17,6 +17,7 @@ HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner +HADOOP_IMAGE=apache/hadoop HADOOP_VERSION=${hadoop.version} OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} OZONE_IMAGE=${docker.ozone.image} diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml index 32059140ce9..0f7105b8ccf 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml @@ -36,7 +36,7 @@ services: - ../_keytabs:/etc/security/keytabs command: [ "krb5kdc","-n" ] kms: - image: apache/hadoop:${HADOOP_VERSION} + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: kms dns_search: . ports: diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml index 5c57eae5145..fceb8e6eb3c 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml @@ -36,7 +36,7 @@ services: - ../_keytabs:/etc/security/keytabs command: [ "krb5kdc","-n" ] kms: - image: apache/hadoop:${HADOOP_VERSION} + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: kms dns_search: . ports: