HDDS-9031. Upgrade acceptance tests to Docker Compose v2 (apache#6667)

hadoop-ozone/dist/src/main/compose/compose_v2_compatibility.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# use Docker Compose v2 if available
+if docker compose version > /dev/null 2>&1; then
+  echo "Using Docker Compose v2"
+  docker-compose() {
+    docker compose --progress quiet "$@"
+  }
+fi

hadoop-ozone/dist/src/main/compose/ozone/run.sh

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+source ../compose_v2_compatibility.sh
+
 declare -i OZONE_DATANODES OZONE_REPLICATION_FACTOR OZONE_SAFEMODE_MIN_DATANODES
 
 ORIG_DATANODES="${OZONE_DATANODES:-}"

hadoop-ozone/dist/src/main/compose/ozonescripts/README.md

@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
-# start-ozone environment
+# ozonescripts environment
 
 This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
 

hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh

@@ -14,4 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
+
+source ../compose_v2_compatibility.sh
+
+docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa

hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh

@@ -14,6 +14,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+source ../compose_v2_compatibility.sh
+
 set -x
 docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1  docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
 docker-compose ps | grep ozonescripts | awk '{print $1}' | xargs -I CONTAINER -n1 docker exec CONTAINER cp /opt/hadoop/etc/hadoop/workers /etc/hadoop/workers

hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh

@@ -14,4 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+source ../compose_v2_compatibility.sh
+
 docker-compose exec -T scm /opt/hadoop/sbin/stop-ozone.sh

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -28,6 +28,8 @@ if [[ -n "${OM_SERVICE_ID}" ]] && [[ "${OM_SERVICE_ID}" != "om" ]]; then
   OM_HA_PARAM="--om-service-id=${OM_SERVICE_ID}"
 fi
 
+source ${_testlib_dir}/compose_v2_compatibility.sh
+
 : ${SCM:=scm}
 
 ## @description create results directory, purging any prior data
@@ -142,11 +144,32 @@ start_docker_env(){
 
   trap stop_docker_env EXIT HUP INT TERM
 
-  docker-compose --ansi never up -d --scale datanode="${datanode_count}"
+  opts=""
+  if has_scalable_datanode; then
+    opts="--scale datanode=${datanode_count}"
+  fi
+
+  docker-compose --ansi never up -d $opts
+
   wait_for_safemode_exit
   wait_for_om_leader
 }
 
+has_scalable_datanode() {
+  local files="${COMPOSE_FILE:-docker-compose.yaml}"
+  local oifs=${IFS}
+  local rc=1
+  IFS=:
+  for f in ${files}; do
+    if [[ -e "${f}" ]] && grep -q '^\s*datanode:' ${f}; then
+      rc=0
+      break
+    fi
+  done
+  IFS=${oifs}
+  return ${rc}
+}
+
 ## @description  Execute robot tests in a specific container.
 ## @param        Name of the container in the docker-compose file
 ## @param        robot test file or directory relative to the smoketest dir
@@ -189,7 +212,7 @@ execute_robot_test(){
       "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
-  FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+  FULL_CONTAINER_NAME=$(docker-compose ps | grep "[-_]${CONTAINER}[-_]" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
   if [[ ${rc} -gt 0 ]] && [[ ${rc} -le 250 ]]; then
@@ -228,7 +251,7 @@ create_stack_dumps() {
 ## @description Copy any 'out' files for daemon processes to the result dir
 copy_daemon_logs() {
   local c f
-  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}[_-]" | awk '{print $1}'); do
     for f in $(docker exec "${c}" ls -1 /var/log/hadoop 2> /dev/null | grep -F -e '.out' -e audit); do
       docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
     done