fix(testing): improve make start-dev-env cluster setup

Fixes #3654

testing/Makefile

@@ -4,7 +4,8 @@ COMPOSE      := docker-compose
 CQLSH        := $(COMPOSE) exec scylla-manager-db cqlsh
 CQLSH_NODE   := $(COMPOSE) exec -T dc1_node_1 cqlsh
 NODETOOL     := $(COMPOSE) exec -T dc1_node_1 nodetool
-SM_NODETOOL := $(COMPOSE) exec -T scylla-manager-db nodetool
+SECOND_NODETOOL := $(COMPOSE) exec -T second_cluster_dc1_node_1 nodetool
+SM_NODETOOL  := $(COMPOSE) exec -T scylla-manager-db nodetool
 YQ           := ../bin/yq
 CURRENT_UID  := $(shell id -u)
 CURRENT_GID  := $(shell id -g)
@@ -28,7 +29,7 @@ ifeq ($(IP_FAMILY), IPV6)
 	MINIO_CERT_DIR := ./minio/certs_ipv6
 endif
 
-SCYLLA_ARGS := --smp 2 --memory 2G --seeds $(SECOND_NET)11,$(SECOND_NET)21
+SCYLLA_ARGS := --smp 2 --memory 2G --seeds $(SECOND_NET)11
 SCYLLA_SECOND_CLUSTER_ARGS := --smp 2 --memory 2G --seeds $(SECOND_NET)31
 
 export SCYLLA_ARGS
@@ -71,24 +72,24 @@ else
 endif
 	@echo "==> Starting containers"
 	mkdir -p $(MINIO_DATA_DIR)
-	@. ./.env && CURRENT_UID=$(CURRENT_UID) CURRENT_GID=$(CURRENT_GID) $(COMPOSE) -f docker-compose.yaml -f $(COMPOSE_FILE) up -d dc1_node_1 dc2_node_1 second_cluster_dc1_node_1
+
+	@. ./.env && CURRENT_UID=$(CURRENT_UID) CURRENT_GID=$(CURRENT_GID) $(COMPOSE) -f docker-compose.yaml -f $(COMPOSE_FILE) up -d dc1_node_1 second_cluster_dc1_node_1
 	$(COMPOSE) exec -T --privileged dc1_node_1 sudo bash -c 'echo "fs.aio-max-nr = 1048579" > /etc/sysctl.d/50-scylla.conf'
 	$(COMPOSE) exec -T --privileged dc1_node_1 sudo sysctl -p /etc/sysctl.d/50-scylla.conf
-	@echo "==> Waiting for first nodes from DC1 and DC2 to spin up"
-	@until [ 2 -le $$($(NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""
-	@echo "==> First nodes from DC1 and DC2 are ready"
-	@. ./.env && CURRENT_UID=$(CURRENT_UID) CURRENT_GID=$(CURRENT_GID) $(COMPOSE) -f docker-compose.yaml -f $(COMPOSE_FILE) up -d dc1_node_2 dc2_node_2 second_cluster_dc1_node_2
-	@echo "==> Waiting for second nodes from DC1 and DC2 to spin up"
-	@until [ 4 -le $$($(NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""
-	@echo "==> Second nodes from DC1 and DC2 are ready"
+	@echo "==> Waiting for dc1 node1"
+	@until [ 1 -le $$($(NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""
+	@until [ 1 -le $$($(SECOND_NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""
+	@. ./.env && CURRENT_UID=$(CURRENT_UID) CURRENT_GID=$(CURRENT_GID) $(COMPOSE) -f docker-compose.yaml -f $(COMPOSE_FILE) up -d second_cluster_dc1_node_2
+
+	@cnt=1; for node in dc1_node_2 dc1_node_3 dc2_node_1 dc2_node_2 dc2_node_3; do \
+  		. ./.env && CURRENT_UID=$(CURRENT_UID) CURRENT_GID=$(CURRENT_GID) $(COMPOSE) -f docker-compose.yaml -f $(COMPOSE_FILE) up -d $$node; \
+		cnt=$$(expr $$cnt + 1); \
+		echo "==> Waiting node $$node number $$cnt"; \
+		until [ $$cnt -le $$($(NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""; done
+
 	@. ./.env && CURRENT_UID=$(CURRENT_UID) CURRENT_GID=$(CURRENT_GID) $(COMPOSE) -f docker-compose.yaml -f $(COMPOSE_FILE) up -d
-	@echo "==> Waiting for sm db cluster"
+	@echo "==> Waiting for the rest of containers"
 	@until [ 1 -le $$($(SM_NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""
-	@echo "==> SM cluster is ready"
-
-	@echo "==> Waiting for rest of the cluster"
-	@until [ 6 -le $$($(NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo ""
-	@echo "==> Cluster is ready"
 
 	@./nodes_exec "rm /root/.cqlshrc || true"
 	@echo "==> Adding Minio user"