scylladb · amnonh · Feb 11, 2024 · Feb 11, 2024 · Feb 11, 2024
diff --git a/kill-all.sh b/kill-all.sh
@@ -1,11 +1,49 @@
 #!/usr/bin/env bash
-
+. versions.sh
 usage="$(basename "$0") [-h] [-g grafana port ] [ -p prometheus port ] [-m alertmanager port] [-w max wait time for prometheus] -- kills existing Grafana and Prometheus Docker instances at given ports"
 GRAFANA_PORT=""
 PROMETHEUS_PORT=""
 ALERTMANAGER_PORT=""
 PROMETHEUS_NAME="aprom"
 PROMETHEUS_KILL_WAITTIME="120"
+LOKI_PORT=""
+PROMTAIL_PORT=""
+for arg; do
+    shift
+    if [ -z "$LIMIT" ]; then
+       case $arg in
+            (--loki-port)
+                LIMIT="1"
+                PARAM="loki-port"
+                ;;
+            (--promtail-port)
+                LIMIT="1"
+                PARAM="promtail-port"
+                ;;
+            (--stack)
+                LIMIT="1"
+                PARAM="stack"
+                ;;
+            (*) set -- "$@" "$arg"
+                ;;
+        esac
+    else
+        DOCR=`echo $arg|cut -d',' -f1`
+        VALUE=`echo $arg|cut -d',' -f2-|sed 's/#/ /g'`
+        NOSPACE=`echo $arg|sed 's/ /#/g'`
+        if [ "$PARAM" = "loki-port" ]; then
+            LOKI_PORT="-p $NOSPACE"
+            unset PARAM
+        elif [ "$PARAM" = "promtail-port" ]; then
+            PROMTAIL_PORT="-p $NOSPACE"
+            unset PARAM
+        elif [ "$PARAM" = "stack" ]; then
+            STACK_ID="$NOSPACE"
+            unset PARAM
+        fi
+        unset LIMIT
+    fi
+done
 while getopts ':hg:p:w:m:' option; do
   case "$option" in
     h) echo "$usage"
@@ -31,6 +69,13 @@ while getopts ':hg:p:w:m:' option; do
   esac
 done
 
+if [ "$STACK_ID" != "" ]; then
+    PROMETHEUS_PORT="-p "${STACK_PROMETHEUS["$STACK_ID"]}
+    GRAFANA_PORT="-p "${STACK_GRAFANA["$STACK_ID"]}
+    ALERTMANAGER_PORT="-p "${STACK_ALERTMANAGER["$STACK_ID"]}
+    PROMETHEUS_NAME="aprom-"${STACK_PROMETHEUS["$STACK_ID"]}
+fi
+
 docker exec $PROMETHEUS_NAME kill 15
 TRIES=0
 OK=0
@@ -51,12 +96,13 @@ sleep 2
 ./kill-container.sh $PROMETHEUS_PORT -b aprom
 ./kill-container.sh $GRAFANA_PORT -b agraf
 ./kill-container.sh $ALERTMANAGER_PORT -b aalert
-./kill-container.sh -b agrafrender
-./kill-container.sh -b vmalert
-./kill-container.sh -b loki
-./kill-container.sh -b promtail
-./kill-container.sh -b sidecar1
-./kill-container.sh -b thanos
-./kill-container.sh -b datadog-agent
-
+if [ -z $STACK_ID ]; then
+    ./kill-container.sh -b agrafrender
+    ./kill-container.sh -b vmalert
+    ./kill-container.sh $LOKI_PORT -b loki
+    ./kill-container.sh $PROMTAIL_PORT -b promtail
+    ./kill-container.sh -b sidecar1
+    ./kill-container.sh -b thanos
+    ./kill-container.sh -b datadog-agent
+fi
 
diff --git a/start-all.sh b/start-all.sh
@@ -64,7 +64,7 @@ elif [[ $(uname) == "Darwin" ]]; then
 fi
 
 function usage {
-  __usage="Usage: $(basename $0) [-h] [--version] [-e] [-d Prometheus data-dir] [-L resolve the servers from the manager running on the given address] [-G path to grafana data-dir] [-s scylla-target-file] [-n node-target-file] [-l] [-v comma separated versions] [-j additional dashboard to load to Grafana, multiple params are supported] [-c grafana environment variable, multiple params are supported] [-b Prometheus command line options] [-g grafana port ] [ -p prometheus port ] [-a admin password] [-m alertmanager port] [ -M scylla-manager version ] [-D encapsulate docker param] [-r alert-manager-config] [-R prometheus-alert-file] [-N manager target file] [-A bind-to-ip-address] [-C alertmanager commands] [-Q Grafana anonymous role (Admin/Editor/Viewer)] [-S start with a system specific dashboard set] [-T additional-prometheus-targets] [--no-loki] [--auto-restart] [--no-renderer] [-f alertmanager-dir]
+  __usage="Usage: $(basename $0) [-h] [--version] [-e] [-d Prometheus data-dir] [-L resolve the servers from the manager running on the given address] [-G path to grafana data-dir] [-s scylla-target-file] [-n node-target-file] [-l] [-v comma separated versions] [-j additional dashboard to load to Grafana, multiple params are supported] [-c grafana environment variable, multiple params are supported] [-b Prometheus command line options] [-g grafana port ] [ -p prometheus port ] [-a admin password] [-m alertmanager port] [ -M scylla-manager version ] [-D encapsulate docker param] [-r alert-manager-config] [-R prometheus-alert-file] [-N manager target file] [-A bind-to-ip-address] [-C alertmanager commands] [-Q Grafana anonymous role (Admin/Editor/Viewer)] [-S start with a system specific dashboard set] [-T additional-prometheus-targets] [--no-loki] [--loki-port port] [--promtail-port port] [--auto-restart] [--no-renderer] [-f alertmanager-dir]
 
 Options:
   -h print this help and exit
@@ -98,6 +98,9 @@ Options:
   -T path/to/prometheus-targets  - Adds additional Prometheus target files.
   -k path/to/loki/storage        - When set, will use the given directory for Loki's data
   --no-loki                      - If set, do not run Loki and promtail.
+  --loki-port port               - If set, loki would use the given port number
+  --promtail-port port           - If set, promtail would use the given port number
+  --promtail-binary-port port    - If set, promtail would use the given port number for the binary protocol
   --no-cas                       - If set, Prometheus will drop all cas related metrics while scrapping
   --no-cdc                       - If set, Prometheus will drop all cdc related metrics while scrapping
   --auto-restart                 - If set, auto restarts the containers on failure.
@@ -107,6 +110,7 @@ Options:
   --enable-protobuf              - If set, enable the experimental Prometheus Protobuf with Native histograms support.
   --target-directory             - If set, prometheus/targets/ directory will be set as a root directory for the target files
                                    the file names should be scylla_server.yml, node_exporter_servers.yml, and  scylla_manager_servers.yml
+  --stack id                     - Use this option when running a secondary stack, id could be 1-4
   --limit container,param        - Allow to set a specific Docker parameter for a container, where container can be:
                                    prometheus, grafana, alertmanager, loki, sidecar, grafanarender
   --archive                      - Treat data directory as an archive. This disables Prometheus time-to-live (infinite retention).
@@ -199,6 +203,9 @@ fi
 if [ -z "$LOKI_DIR" ]; then
   LOKI_DIR=""
 fi
+if [ -z "$LOKI_PORT" ]; then
+  LOKI_PORT=""
+fi
 LIMITS=""
 VOLUMES=""
 PARAMS=""
@@ -215,6 +222,18 @@ for arg; do
        case $arg in
             (--no-loki) RUN_LOKI=0
                 ;;
+            (--loki-port)
+                LIMIT="1"
+                PARAM="loki-port"
+                ;;
+            (--promtail-port)
+                LIMIT="1"
+                PARAM="promtail-port"
+                ;;
+            (--promtail-binary-port)
+                LIMIT="1"
+                PARAM="promtail-binary-port"
+                ;;
             (--no-renderer) RUN_RENDERER=""
                 ;;
             (--thanos-sc) RUN_THANOS_SC=1
@@ -261,6 +280,10 @@ for arg; do
                 LIMIT="1"
                 PARAM="datadog-hostname"
                 ;;
+            (--stack)
+                LIMIT="1"
+                PARAM="stack"
+                ;;
             (--no-cas-cdc)
                 PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-cas-cdc"
                 ;;
@@ -308,6 +331,18 @@ for arg; do
         elif [ "$PARAM" = "datadog-hostname" ]; then
             DATDOGPARAM="$DATDOGPARAM -H $NOSPACE"
             unset PARAM
+        elif [ "$PARAM" = "loki-port" ]; then
+            LOKI_PORT="$LOKI_PORT -p $NOSPACE"
+            unset PARAM
+        elif [ "$PARAM" = "promtail-port" ]; then
+            LOKI_PORT="$LOKI_PORT -t $NOSPACE"
+            unset PARAM
+        elif [ "$PARAM" = "promtail-binary-port" ]; then
+            LOKI_PORT="$LOKI_PORT -T $NOSPACE"
+            unset PARAM
+        elif [ "$PARAM" = "stack" ]; then
+            STACK_ID="$NOSPACE"
+            unset PARAM
         else
             if [ -z "${DOCKER_LIMITS[$DOCR]}" ]; then
                 DOCKER_LIMITS[$DOCR]=""
@@ -508,6 +543,16 @@ else
         VERSIONS=$ALL
     fi
 fi
+if [ $STACK_ID != "" ]; then
+    echo "Running a seconddary stack $STACK_ID"
+    echo "Note that the following containers will not run: loki, promtail, grafana renderer"
+    echo "to stop it use ./kill-all.sh --stack $STACK_ID"
+    RUN_LOKI=0
+    RUN_RENDERER=""
+    PROMETHEUS_PORT=${STACK_PROMETHEUS["$STACK_ID"]}
+    GRAFANA_PORT="-g"${STACK_GRAFANA["$STACK_ID"]}
+    ALERTMANAGER_PORT="-p "${STACK_ALERTMANAGER["$STACK_ID"]}
+fi
 
 ALERTMANAGER_COMMAND=""
 for val in "${ALERTMANAGER_COMMANDS[@]}"; do
@@ -523,7 +568,7 @@ fi
 LOKI_ADDRESS=""
 if [ $RUN_LOKI -eq 1 ]; then
     echo "Wait for Loki container to start."
-	LOKI_ADDRESS=`./start-loki.sh $BIND_ADDRESS_CONFIG $LOKI_DIR -D "$DOCKER_PARAM" $LIMITS $VOLUMES $PARAMS -m $AM_ADDRESS`
+	LOKI_ADDRESS=`./start-loki.sh $BIND_ADDRESS_CONFIG $LOKI_DIR $LOKI_PORT -D "$DOCKER_PARAM" $LIMITS $VOLUMES $PARAMS -m $AM_ADDRESS`
 	if [ $? -ne 0 ]; then
 	    echo "$LOKI_ADDRESS"
 	    exit 1

diff --git a/start-loki.sh b/start-loki.sh
@@ -70,13 +70,17 @@ for arg; do
         unset LIMIT
     fi
 done
-while getopts ':hlp:D:m:A:k:' option; do
+while getopts ':hlp:D:m:A:k:t:T:' option; do
   case "$option" in
     h) echo "$usage"
        exit
        ;;
     p) LOKI_PORT=$OPTARG
        ;;
+    t) PROMTAIL_PORT=$OPTARG
+       ;;
+    T) PROMTAIL_BIN_PORT=$OPTARG
+       ;;
     r) LOKI_RULE_DIR=`readlink -m $OPTARG`
        ;;
     l) DOCKER_PARAM="$DOCKER_PARAM --net=host"
@@ -176,6 +180,9 @@ if [ -z $PROMTAIL_PORT ]; then
 else
     PROMTAIL_NAME=promtail-$PROMTAIL_PORT
 fi
+if [ -z $PROMTAIL_BIN_PORT ]; then
+    PROMTAIL_BIN_PORT=1514
+fi
 
 docker container inspect $PROMTAIL_NAME > /dev/null 2>&1
 if [ $? -eq 0 ]; then
@@ -184,7 +191,7 @@ if [ $? -eq 0 ]; then
 fi
 
 if [[ ! $DOCKER_PARAM = *"--net=host"* ]]; then
-    PROMTAIL_PORT_MAPPING="-p $BIND_ADDRESS$PROMTAIL_PORT:9080 -p ${BIND_ADDRESS}1514:1514"
+    PROMTAIL_PORT_MAPPING="-p $BIND_ADDRESS$PROMTAIL_PORT:9080 -p ${BIND_ADDRESS}$PROMTAIL_BIN_PORT:1514"
 fi
 
 sed "s/LOKI_IP/$LOKI_ADDRESS/" loki/promtail/promtail_config.template.yml > loki/promtail/promtail_config.yml

diff --git a/versions.sh b/versions.sh
@@ -5,13 +5,31 @@ unset MANAGER_DEFAULT_VERSION
 unset DEFAULT_ENTERPRISE_VERSION
 unset DOCKER_LIMITS
 unset DOCKER_PARAMS
+unset STACK_PROMETHEUS
+unset STACK_GRAFANA
+unset STACK_ALERTMANAGER
 declare -A SUPPORTED_VERSIONS
 declare -A MANAGER_SUPPORTED_VERSIONS
 declare -A DEFAULT_VERSION
 declare -A MANAGER_DEFAULT_VERSION
 declare -A DEFAULT_ENTERPRISE_VERSION
 declare -A DOCKER_LIMITS
 declare -A DOCKER_PARAMS
+declare -A STACK_PROMETHEUS
+declare -A STACK_GRAFANA
+declare -A STACK_ALERTMANAGER
+STACK_PROMETHEUS["1"]="9051"
+STACK_GRAFANA["1"]="3001"
+STACK_ALERTMANAGER["1"]="9041"
+STACK_PROMETHEUS["2"]="9052"
+STACK_GRAFANA["2"]="3002"
+STACK_ALERTMANAGER["2"]="9042"
+STACK_PROMETHEUS["3"]="9053"
+STACK_GRAFANA["3"]="3003"
+STACK_ALERTMANAGER["3"]="9043"
+STACK_PROMETHEUS["4"]="9054"
+STACK_GRAFANA["4"]="3004"
+STACK_ALERTMANAGER["4"]="9044"
 SUPPORTED_VERSIONS["3.8"]="4.3,4.4,4.5,2020.1,2021.1,master"
 MANAGER_SUPPORTED_VERSIONS["3.8"]="2.3,2.4,2.5"
 DEFAULT_VERSION["3.8"]="4.4"