HDDS-9031. Upgrade acceptance tests to Docker Compose v2

hadoop-ozone/dist/src/main/compose/ozonescripts/README.md

@@ -12,17 +12,17 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
-# start-ozone environment
+# ozonescripts environment
 
-This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
+This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.  It requires Docker Compose v2.
 
 There are ssh connections between the containers and the start/stop scripts could handle the start/stop process
 similar to a real cluster.
 
 To use it, first start the cluster:
 
 ```
-docker-compose up -d
+docker compose up -d
 ```
 
 After a successful startup (which starts only the ssh daemons) you can start ozone:

hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh

@@ -14,4 +14,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
+docker compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa

hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh

@@ -15,12 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -x
-docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1  docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
-docker-compose ps | grep ozonescripts | awk '{print $1}' | xargs -I CONTAINER -n1 docker exec CONTAINER cp /opt/hadoop/etc/hadoop/workers /etc/hadoop/workers
-docker-compose exec -T scm /opt/hadoop/bin/ozone scm --init
-docker-compose exec -T scm /opt/hadoop/sbin/start-ozone.sh
+docker compose ps | grep datanode | awk '{print $1}' | xargs -n1  docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
+docker compose ps | grep ozonescripts | awk '{print $1}' | xargs -I CONTAINER -n1 docker exec CONTAINER cp /opt/hadoop/etc/hadoop/workers /etc/hadoop/workers
+docker compose exec -T scm /opt/hadoop/bin/ozone scm --init
+docker compose exec -T scm /opt/hadoop/sbin/start-ozone.sh
 #We need a running SCM for om objectstore creation
 #TODO create a utility to wait for the startup
 sleep 10
-docker-compose exec -T om /opt/hadoop/bin/ozone om --init
-docker-compose exec -T scm /opt/hadoop/sbin/start-ozone.sh
+docker compose exec -T om /opt/hadoop/bin/ozone om --init
+docker compose exec -T scm /opt/hadoop/sbin/start-ozone.sh

hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh

@@ -14,4 +14,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-docker-compose exec -T scm /opt/hadoop/sbin/stop-ozone.sh
+docker compose exec -T scm /opt/hadoop/sbin/stop-ozone.sh

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -28,6 +28,14 @@ if [[ -n "${OM_SERVICE_ID}" ]] && [[ "${OM_SERVICE_ID}" != "om" ]]; then
   OM_HA_PARAM="--om-service-id=${OM_SERVICE_ID}"
 fi
 
+# use Docker Compose v2 if available
+if docker compose version > /dev/null 2>&1; then
+  echo "Using Docker Compose v2"
+  docker-compose() {
+    docker compose --progress quiet "$@"
+  }
+fi
+
 : ${SCM:=scm}
 
 ## @description create results directory, purging any prior data
@@ -142,11 +150,32 @@ start_docker_env(){
 
   trap stop_docker_env EXIT HUP INT TERM
 
-  docker-compose --ansi never up -d --scale datanode="${datanode_count}"
+  opts=""
+  if has_scalable_datanode; then
+    opts="--scale datanode=${datanode_count}"
+  fi
+
+  docker-compose --ansi never up -d $opts
+
   wait_for_safemode_exit
   wait_for_om_leader
 }
 
+has_scalable_datanode() {
+  local files="${COMPOSE_FILE:-docker-compose.yaml}"
+  local oifs=${IFS}
+  local rc=1
+  IFS=:
+  for f in ${files}; do
+    if [[ -e "${f}" ]] && grep -q '^\s*datanode:' ${f}; then
+      rc=0
+      break
+    fi
+  done
+  IFS=${oifs}
+  return ${rc}
+}
+
 ## @description  Execute robot tests in a specific container.
 ## @param        Name of the container in the docker-compose file
 ## @param        robot test file or directory relative to the smoketest dir
@@ -189,7 +218,7 @@ execute_robot_test(){
       "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
-  FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+  FULL_CONTAINER_NAME=$(docker-compose ps | grep "[-_]${CONTAINER}[-_]" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
   if [[ ${rc} -gt 0 ]] && [[ ${rc} -le 250 ]]; then
@@ -228,7 +257,7 @@ create_stack_dumps() {
 ## @description Copy any 'out' files for daemon processes to the result dir
 copy_daemon_logs() {
   local c f
-  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}[_-]" | awk '{print $1}'); do
     for f in $(docker exec "${c}" ls -1 /var/log/hadoop 2> /dev/null | grep -F -e '.out' -e audit); do
       docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
     done