From de228993e6c8dd65b0f0062b292fd767787b04f4 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com>
Date: Thu, 16 May 2024 09:27:40 +0200
Subject: [PATCH] HDDS-9031. Upgrade acceptance tests to Docker Compose v2
 (#6667)

---
 .../main/compose/compose_v2_compatibility.sh  | 24 ++++++++++++++
 .../dist/src/main/compose/ozone/run.sh        |  2 ++
 .../src/main/compose/ozonescripts/README.md   |  2 +-
 .../dist/src/main/compose/ozonescripts/ps.sh  |  5 ++-
 .../src/main/compose/ozonescripts/start.sh    |  3 ++
 .../src/main/compose/ozonescripts/stop.sh     |  3 ++
 hadoop-ozone/dist/src/main/compose/testlib.sh | 29 +++++++++++++++--
 .../dist/src/main/smoketest/env-compose.robot | 32 -------------------
 8 files changed, 63 insertions(+), 37 deletions(-)
 create mode 100644 hadoop-ozone/dist/src/main/compose/compose_v2_compatibility.sh
 delete mode 100644 hadoop-ozone/dist/src/main/smoketest/env-compose.robot

diff --git a/hadoop-ozone/dist/src/main/compose/compose_v2_compatibility.sh b/hadoop-ozone/dist/src/main/compose/compose_v2_compatibility.sh
new file mode 100644
index 00000000000..b098521b528
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/compose_v2_compatibility.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# use Docker Compose v2 if available
+if docker compose version > /dev/null 2>&1; then
+  echo "Using Docker Compose v2"
+  docker-compose() {
+    docker compose --progress quiet "$@"
+  }
+fi
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/run.sh b/hadoop-ozone/dist/src/main/compose/ozone/run.sh
index 671d2cda6c9..c29ca8a1e04 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/run.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/run.sh
@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+source ../compose_v2_compatibility.sh
+
 declare -i OZONE_DATANODES OZONE_REPLICATION_FACTOR OZONE_SAFEMODE_MIN_DATANODES
 
 ORIG_DATANODES="${OZONE_DATANODES:-}"
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md b/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md
index e191825f633..36480239af6 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md
@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
-# start-ozone environment
+# ozonescripts environment
 
 This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
 
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh
index d5e2c386752..830d03595fd 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh
@@ -14,4 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
\ No newline at end of file
+
+source ../compose_v2_compatibility.sh
+
+docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
index 2ce768dc785..cea7722c5d2 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
@@ -14,6 +14,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+source ../compose_v2_compatibility.sh
+
 set -x
 docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1  docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
 docker-compose ps | grep ozonescripts | awk '{print $1}' | xargs -I CONTAINER -n1 docker exec CONTAINER cp /opt/hadoop/etc/hadoop/workers /etc/hadoop/workers
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh
index 012fffb75cf..743a0d12a5c 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh
@@ -14,4 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+source ../compose_v2_compatibility.sh
+
 docker-compose exec -T scm /opt/hadoop/sbin/stop-ozone.sh
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 5ac3d09d59a..d4ad22e58ec 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -28,6 +28,8 @@ if [[ -n "${OM_SERVICE_ID}" ]] && [[ "${OM_SERVICE_ID}" != "om" ]]; then
   OM_HA_PARAM="--om-service-id=${OM_SERVICE_ID}"
 fi
 
+source ${_testlib_dir}/compose_v2_compatibility.sh
+
 : ${SCM:=scm}
 
 ## @description create results directory, purging any prior data
@@ -142,11 +144,32 @@ start_docker_env(){
 
   trap stop_docker_env EXIT HUP INT TERM
 
-  docker-compose --ansi never up -d --scale datanode="${datanode_count}"
+  opts=""
+  if has_scalable_datanode; then
+    opts="--scale datanode=${datanode_count}"
+  fi
+
+  docker-compose --ansi never up -d $opts
+
   wait_for_safemode_exit
   wait_for_om_leader
 }
 
+has_scalable_datanode() {
+  local files="${COMPOSE_FILE:-docker-compose.yaml}"
+  local oifs=${IFS}
+  local rc=1
+  IFS=:
+  for f in ${files}; do
+    if [[ -e "${f}" ]] && grep -q '^\s*datanode:' ${f}; then
+      rc=0
+      break
+    fi
+  done
+  IFS=${oifs}
+  return ${rc}
+}
+
 ## @description  Execute robot tests in a specific container.
 ## @param        Name of the container in the docker-compose file
 ## @param        robot test file or directory relative to the smoketest dir
@@ -189,7 +212,7 @@ execute_robot_test(){
       "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
-  FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+  FULL_CONTAINER_NAME=$(docker-compose ps | grep "[-_]${CONTAINER}[-_]" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
   if [[ ${rc} -gt 0 ]] && [[ ${rc} -le 250 ]]; then
@@ -228,7 +251,7 @@ create_stack_dumps() {
 ## @description Copy any 'out' files for daemon processes to the result dir
 copy_daemon_logs() {
   local c f
-  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}[_-]" | awk '{print $1}'); do
     for f in $(docker exec "${c}" ls -1 /var/log/hadoop 2> /dev/null | grep -F -e '.out' -e audit); do
       docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
     done
diff --git a/hadoop-ozone/dist/src/main/smoketest/env-compose.robot b/hadoop-ozone/dist/src/main/smoketest/env-compose.robot
deleted file mode 100644
index d21eacaea50..00000000000
--- a/hadoop-ozone/dist/src/main/smoketest/env-compose.robot
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation       High level utilities to execute commands and tests in docker-compose based environments.
-Resource            commonlib.robot
-Test Timeout        5 minutes
-
-*** Keywords ***
-
-Run tests on host
-    [arguments]        ${host}       ${robotfile}
-    ${result} =        Execute       docker-compose exec ${host} robot smoketest/${robotfile}
-
-Execute on host
-    [arguments]                     ${host}     ${command}
-    ${rc}                           ${output} =                 Run And Return Rc And Output           docker-compose exec ${host} ${command}
-    Log                             ${output}
-    Should Be Equal As Integers     ${rc}                       0
-    [return]                        ${output}