From 5dd625a718f722ea627936ec34f4c87f078f07bb Mon Sep 17 00:00:00 2001 From: Lenin Mehedy Date: Wed, 4 Oct 2023 16:52:46 +1100 Subject: [PATCH] feat: add support for scheduling pods based on node labels, taints, and affinity (#352) Signed-off-by: Lenin Mehedy --- .github/workflows/zxc-compile-code.yaml | 3 +- charts/hedera-network/Chart.lock | 4 +- charts/hedera-network/Chart.yaml | 8 ++-- .../templates/network-node-statefulset.yaml | 22 ++++++++++ charts/hedera-network/templates/pdb.yaml | 24 +++++++++++ .../templates/rbac/pod-monitor.yaml | 2 + charts/hedera-network/tests/env.sh | 15 +++++-- charts/hedera-network/tests/env.template | 3 ++ charts/hedera-network/tests/run.sh | 2 +- charts/hedera-network/values.yaml | 43 +++++++++++++++++-- dev/Makefile | 28 ++++-------- dev/dev-cluster.yaml | 8 ++++ dev/scripts/docker.sh | 6 ++- dev/scripts/env.sh | 21 ++++++++- dev/scripts/gateway.sh | 20 +++++++++ dev/scripts/main.sh | 25 ++++++++--- dev/scripts/telemetry.sh | 12 ++++++ 17 files changed, 206 insertions(+), 40 deletions(-) create mode 100644 charts/hedera-network/templates/pdb.yaml create mode 100644 dev/dev-cluster.yaml diff --git a/.github/workflows/zxc-compile-code.yaml b/.github/workflows/zxc-compile-code.yaml index 788389b28..922a50e3c 100644 --- a/.github/workflows/zxc-compile-code.yaml +++ b/.github/workflows/zxc-compile-code.yaml @@ -102,7 +102,7 @@ jobs: uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 if: ${{ inputs.enable-unit-tests && !cancelled() }} with: - cluster_name: fst + config: dev/dev-cluster.yaml version: v0.19.0 verbosity: 3 wait: 120s @@ -125,6 +125,7 @@ jobs: run: | kubectl config get-contexts kubectl get crd + kubectl get node --show-labels # This step is currently required because the Hedera Services artifacts are not publicly accessible. # May be removed once the artifacts are publicly accessible. diff --git a/charts/hedera-network/Chart.lock b/charts/hedera-network/Chart.lock index 6d168fe77..b32b63749 100644 --- a/charts/hedera-network/Chart.lock +++ b/charts/hedera-network/Chart.lock @@ -8,5 +8,5 @@ dependencies: - name: tenant repository: https://operator.min.io/ version: 5.0.7 -digest: sha256:cf355b295abceb5814ef57d3e146ec9d4e8db7365a700079d683bd5f766ad374 -generated: "2023-09-20T13:51:41.203996+10:00" +digest: sha256:5dbc1a4af8f2b057dbd7730b6308e1a2954f3f95f86e8484bb232e64ed12e923 +generated: "2023-10-04T15:47:44.747012+11:00" diff --git a/charts/hedera-network/Chart.yaml b/charts/hedera-network/Chart.yaml index 02fb107a1..a55d561ba 100644 --- a/charts/hedera-network/Chart.yaml +++ b/charts/hedera-network/Chart.yaml @@ -24,18 +24,20 @@ version: 0.8.0 appVersion: "0.8.0" # This is range of versions of Kubernetes server that is supported by this chart. -kubeVersion: ">=1.25.0" +# Note we need to use -0 suffix to support GKE version +# Reference: https://github.com/helm/helm/issues/3810#issuecomment-379877753 +kubeVersion: ">=1.25.0-0" dependencies: - name: hedera-explorer version: 0.2.0 - condition: cloud.minio.enable + condition: hedera-explorer.enable - name: hedera-mirror alias: hedera-mirror-node version: 0.86.0 repository: https://hashgraph.github.io/hedera-mirror-node/charts - condition: cloud.minio.enable + condition: hedera-mirror-node.enable - name: tenant alias: minio-server diff --git a/charts/hedera-network/templates/network-node-statefulset.yaml b/charts/hedera-network/templates/network-node-statefulset.yaml index 50f00ef4d..ac90a2b9f 100644 --- a/charts/hedera-network/templates/network-node-statefulset.yaml +++ b/charts/hedera-network/templates/network-node-statefulset.yaml @@ -17,6 +17,13 @@ metadata: namespace: {{ default $.Release.Namespace $.Values.global.namespaceOverride }} labels: app: network-{{ $node.name }} + {{- if $.Values.deployment.podLabels }} + {{- $.Values.deployment.podLabels | toYaml | nindent 4 }} + {{- end }} + {{- if $.Values.deployment.podAnnotations }} + annotations: + {{- $.Values.deployment.podAnnotations | toYaml | nindent 4 }} + {{- end }} spec: replicas: 1 serviceName: "network-{{ $node.name }}" @@ -30,6 +37,21 @@ spec: fullstack.hedera.com/type: network-node fullstack.hedera.com/node-name: {{ $node.name }} spec: + {{- if $.Values.deployment.nodeSelectors }} + nodeSelector: + {{- $.Values.deployment.nodeSelectors | toYaml | nindent 8 }} + {{- end }} + {{- if $.Values.deployment.tolerations }} + tolerations: + {{- $.Values.deployment.tolerations | toYaml | nindent 8 }} + {{- end }} + {{- if $.Values.deployment.affinity }} + affinity: + {{- $.Values.deployment.affinity | toYaml | nindent 8 }} + {{- end }} + {{- if $.Values.deployment.priorityClassName }} + priorityClassName: {{ $.Values.deployment.priorityClassName }} + {{- end }} terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }} volumes: - name: hgcapp-storage # change me diff --git a/charts/hedera-network/templates/pdb.yaml b/charts/hedera-network/templates/pdb.yaml new file mode 100644 index 000000000..643b1f170 --- /dev/null +++ b/charts/hedera-network/templates/pdb.yaml @@ -0,0 +1,24 @@ +{{- if $.Values.deployment.podDisruptionBudget.create }} +{{ range $index, $node := $.Values.hedera.nodes }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: network-node-pdb-{{ $node.name }} + namespace: {{ default $.Release.Namespace $.Values.global.namespaceOverride }} + labels: + fullstack.hedera.com/type: pod-disruption-budget + fullstack.hedera.com/node-name: {{ $node.name }} +spec: + selector: + matchLabels: + fullstack.hedera.com/type: network-node + fullstack.hedera.com/node-name: {{ $node.name }} + {{- if $.Values.deployment.podDisruptionBudget.minAvailable }} + minAvailable: {{ $.Values.deployment.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if $.Values.deployment.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ $.Values.deployment.podDisruptionBudget.maxUnavailable }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/hedera-network/templates/rbac/pod-monitor.yaml b/charts/hedera-network/templates/rbac/pod-monitor.yaml index d6912a9b8..d6a2ec5e7 100644 --- a/charts/hedera-network/templates/rbac/pod-monitor.yaml +++ b/charts/hedera-network/templates/rbac/pod-monitor.yaml @@ -1,3 +1,4 @@ +{{- if $.Values.tester.deployPodMonitor | eq "true" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -17,3 +18,4 @@ roleRef: kind: ClusterRole name: {{ $.Values.tester.clusterRoleName }} apiGroup: rbac.authorization.k8s.io +{{- end }} \ No newline at end of file diff --git a/charts/hedera-network/tests/env.sh b/charts/hedera-network/tests/env.sh index e86a91f40..050672dce 100755 --- a/charts/hedera-network/tests/env.sh +++ b/charts/hedera-network/tests/env.sh @@ -3,8 +3,10 @@ # Every script must load (source) this in the beginning # Warning: avoid making these variables readonly since it can be sourced multiple times +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + # load .env file if it exists in order to load variables with custom values -ENV_FILE="$(dirname "${BASH_SOURCE[0]}")/.env" +ENV_FILE="${CUR_DIR}/.env" if [[ -f "${ENV_FILE}" ]]; then set -a # shellcheck source=./../temp/.env @@ -13,8 +15,15 @@ if [[ -f "${ENV_FILE}" ]]; then fi # set global env variables if not set -BATS_HOME="${BATS_HOME:-../../../dev/bats}" -TESTS_DIR="${TESTS_DIR:-.}" +BATS_HOME="${BATS_HOME:-${CUR_DIR}/../../../dev/bats}" +TESTS_DIR="${TESTS_DIR:-${CUR_DIR}}" + +TOTAL_NODES="${TOTAL_NODES:-3}" +USER="${USER:-changeme}" +NAMESPACE="${NAMESPACE:-fst-${USER}}" +LOG_DIR="${LOG_DIR:-${CUR_DIR}/logs}" +LOG_FILE="${LOG_FILE:-helm-test.log}" +OUTPUT_LOG="${OUTPUT_LOG:-false}" [ ! -d "${LOG_DIR}" ] && mkdir "${LOG_DIR}" echo "--------------------------Env Setup: fullstack-testing Helm Test------------------------------------------------" diff --git a/charts/hedera-network/tests/env.template b/charts/hedera-network/tests/env.template index 1df446a1c..5ed4d1fb1 100644 --- a/charts/hedera-network/tests/env.template +++ b/charts/hedera-network/tests/env.template @@ -1,3 +1,6 @@ +USER="${USER:-changeme}" +NAMESPACE="${NAMESPACE:-fst-${USER}}" + TOTAL_NODES=3 LOG_DIR="${LOG_DIR:-/tmp/fullstack-testing-logs}" diff --git a/charts/hedera-network/tests/run.sh b/charts/hedera-network/tests/run.sh index b872867bf..33e55969d 100755 --- a/charts/hedera-network/tests/run.sh +++ b/charts/hedera-network/tests/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -CUR_DIR=$(dirname "${BASH_SOURCE[0]}") +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${CUR_DIR}/env.sh" source "${CUR_DIR}/logging.sh" diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml index 5d2f1b58a..426cfe8d2 100644 --- a/charts/hedera-network/values.yaml +++ b/charts/hedera-network/values.yaml @@ -5,8 +5,6 @@ global: # cloud configuration cloud: - minio: - enable: true buckets: streamBucket: "fst-streams" backupBucket: "fst-backups" @@ -14,6 +12,8 @@ cloud: enable: "true" gcs: enable: "true" + minio: + enable: true # telemetry configurations telemetry: @@ -26,6 +26,7 @@ terminationGracePeriodSeconds: 10 # helm test container tester: + deployPodMonitor: "true" clusterRoleName: "pod-monitor-role" # this is a shared cluster role for all namespaces image: registry: "ghcr.io" @@ -63,7 +64,6 @@ gatewayApi: route: hostname: "{{ .node.name }}.fst.local" - # default settings for a single node # This default configurations can be overridden for each node in the hedera.nodes section. defaults: @@ -222,7 +222,9 @@ minio-server: certificate: requestAutoCert: false +# hedera mirror node configuration hedera-mirror-node: + enable: true global: namespaceOverride: "{{ tpl (.Values.global.namespaceOverride | toString) }}" # importer is a component of the hedera mirror node @@ -250,7 +252,9 @@ hedera-mirror-node: bucketName: "fst-streams" # for s3 configuration of mirror node look at uploader-mirror-secrets.yaml +# hedera explorer configuration hedera-explorer: + enable: true global: namespaceOverride: "{{ tpl (.Values.global.namespaceOverride | toString) }}" # The hedera explorer UI /api url will proxy all request to mirror node @@ -272,6 +276,39 @@ hedera-explorer: } ] +# common deployment configuration +deployment: + podAnnotations: {} + podLabels: {} + nodeSelectors: + fullstack-scheduling.io/os: linux + fullstack-scheduling.io/role: network + tolerations: + - key: "fullstack-scheduling.io/os" + operator: "Equal" + value: "linux" + effect: "NoSchedule" + - key: "fullstack-scheduling.io/role" + operator: "Equal" + value: "network" + effect: "NoSchedule" + # Specify pod affinity + # Use complete affinity spec starting with key "nodeAffinity:" + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + affinity: {} + priorityClassName: {} + ## PodDisruptionBudget for fullstack testing pods + ## Default backend Pod Disruption Budget configuration + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + ## @param deployment.podDisruptionBudget.create Enable Pod Disruption Budget configuration + ## @param deployment.podDisruptionBudget.minAvailable Minimum number/percentage of pods that should remain scheduled + ## @param deployment.podDisruptionBudget.maxUnavailable Maximum number/percentage of pods that should remain scheduled + ## + podDisruptionBudget: + create: true + minAvailable: 1 + maxUnavailable: "" + # hedera node configuration # Only the name of the node is required. The rest of the configuration will be inherited from `defaults` section hedera: diff --git a/dev/Makefile b/dev/Makefile index e62797fe6..f54cfb98f 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -138,7 +138,10 @@ run-func: source "${SCRIPTS_DIR}/${SCRIPT_NAME}" && ${FUNC} .PHONY: start -start: deploy-minio-operator-if-required update-helm-dependencies deploy-network setup-nodes start-nodes +start: ci-deploy-network setup-nodes start-nodes + +.PHONY: stop +stop: stop-nodes destroy-network .PHONY: restart restart: stop-nodes start-nodes @@ -233,30 +236,17 @@ destroy-test-container: local-kubectl-bats: source "${SCRIPTS_DIR}/${DOCKER_SCRIPT}" && build_kubectl_bats "${CLUSTER_NAME}" -# Here we run all steps in sequence, if any step fails, deploy-all trap the EXIT and run cleanup -.PHONY: run-deploy-seq -run-deploy-seq: setup deploy-network helm-test setup-nodes start-nodes - -.PHONY: deploy-all -deploy-all: +.PHONY: ci-test +ci-test: # Enable cleanup_test function so that even if test fails, we cleanup the cluster. # We are only enabling this in this make target, however if necessary, similar pattern can be used in other targets. # Ref: https://stackoverflow.com/questions/28597794/how-can-i-clean-up-after-an-error-in-a-makefile - function cleanup_test { + # NOTE: It needs latest make (version ~=4.3) + function cleanup_test () { $(MAKE) destroy-network } trap cleanup_test EXIT # always destroy-network on exit - $(MAKE) run-deploy-seq - -.PHONY: destroy-all -destroy-all: - -$(MAKE) destroy-network - -$(MAKE) undeploy-minio-operator - -$(MAKE) destroy-prometheus-operator - -.PHONY: ci-test -ci-test: setup-cluster local-kubectl-bats - $(MAKE) deploy-all CHART_VALUES_FILES="$(PWD)/ci/ci-values.yaml" + $(MAKE) ci-deploy-network setup-nodes start-nodes .PHONY: ci-deploy-network ci-deploy-network: setup-cluster local-kubectl-bats diff --git a/dev/dev-cluster.yaml b/dev/dev-cluster.yaml new file mode 100644 index 000000000..df235cf65 --- /dev/null +++ b/dev/dev-cluster.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: fst # this is overridden if CLUSTER_NAME env var is set. Check .env file +nodes: + - role: control-plane + labels: + fullstack-scheduling.io/os: linux + fullstack-scheduling.io/role: network \ No newline at end of file diff --git a/dev/scripts/docker.sh b/dev/scripts/docker.sh index 1f69ee151..1caf83667 100644 --- a/dev/scripts/docker.sh +++ b/dev/scripts/docker.sh @@ -10,8 +10,10 @@ function build_kubectl_bats() { [[ -z "${CLUSTER_NAME}" ]] && echo "ERROR: [build_kubectl_bats] Cluster name is required" && return 1 echo "" - echo "Building kubectl-bats image" + echo "Building kubectl-bats image" echo "-----------------------------------------------------------------------------------------------------" - cd "${DOCKERFILE_DIR}/kubectl-bats" && docker build -t "${KUBECTL_BATS_IMAGE}" . + cd "${DOCKERFILE_DIR}/kubectl-bats" && docker build -t "${KUBECTL_BATS_IMAGE}" . kind load docker-image "${KUBECTL_BATS_IMAGE}" -n "${CLUSTER_NAME}" + + log_time "build_kubectl_bats" } \ No newline at end of file diff --git a/dev/scripts/env.sh b/dev/scripts/env.sh index b12884910..ba7b11325 100644 --- a/dev/scripts/env.sh +++ b/dev/scripts/env.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +start_time=$(date +%s) + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" readonly SCRIPT_DIR @@ -58,7 +60,11 @@ function setup_kubectl_context() { kubectl get ns echo "Setting kubectl context..." - kubectl config use-context "kind-${CLUSTER_NAME}" + local count + count=$(kubectl config get-contexts --no-headers | grep -c "kind-${CLUSTER_NAME}") + if [[ $count -ne 0 ]]; then + kubectl config use-context "kind-${CLUSTER_NAME}" + fi kubectl config set-context --current --namespace="${NAMESPACE}" kubectl config get-contexts } @@ -68,6 +74,19 @@ function setup() { load_env_file } +function log_time() { + local end_time duration execution_time + + local func_name=$1 + + end_time=$(date +%s) + duration=$((end_time - start_time)) + execution_time=$(printf "%.2f seconds" "${duration}") + echo "-----------------------------------------------------------------------------------------------------" + echo "<<< ${func_name} execution took: ${execution_time} >>>" + echo "-----------------------------------------------------------------------------------------------------" +} + setup echo "--------------------------Env Setup: fullstack-testing ------------------------------------------------" diff --git a/dev/scripts/gateway.sh b/dev/scripts/gateway.sh index 5ac81f032..0817a2c10 100644 --- a/dev/scripts/gateway.sh +++ b/dev/scripts/gateway.sh @@ -21,6 +21,8 @@ function deploy_haproxy_ingress() { echo "HAProxy Ingress Controller is already installed" echo "" fi + + log_time "deploy_haproxy_ingress" } function destroy_haproxy_ingress() { @@ -40,6 +42,8 @@ function destroy_haproxy_ingress() { echo "HAProxy Ingress Controller is uninstalled" echo "" + + log_time "destroy_haproxy_ingress" } function deploy_gateway_api_crd() { @@ -81,6 +85,8 @@ function deploy_envoy_gateway_api() { fi get_gateway_status + + log_time "deploy_envoy_gateway_api" } function get_gateway_status() { @@ -116,6 +122,8 @@ function destroy_envoy_gateway_api() { echo "Envoy Gateway API is uninstalled" echo "" + + log_time "destroy_envoy_gateway_api" } function uninstall_crd() { @@ -129,6 +137,8 @@ function uninstall_crd() { kubectl delete crd "${name}" done fi + + log_time "uninstall_crd" } function expose_envoy_gateway_svc() { @@ -145,6 +155,8 @@ function expose_envoy_gateway_svc() { echo "Exposing Envoy Gateway Service: ${ENVOY_SERVICE} on ${local_port}:${gateway_port}" echo "-----------------------------------------------------------------------------------------------------" kubectl port-forward "svc/${ENVOY_SERVICE}" -n envoy-gateway-system "${local_port}":"${gateway_port}" & + + log_time "expose_envoy_gateway_svc" } function unexpose_envoy_gateway_svc() { @@ -157,6 +169,8 @@ function unexpose_envoy_gateway_svc() { echo "-----------------------------------------------------------------------------------------------------" kill "${GATEWAY_SVC_PID}" &>/dev/null || true fi + + log_time "unexpose_envoy_gateway_svc" } function test_http_route() { @@ -195,6 +209,8 @@ function test_http_route() { echo "-----------------------------------------------------------------------------------------------------" unexpose_envoy_gateway_svc || true kubectl delete -f "${GATEWAY_API_DIR}/http-debug.yaml" + + log_time "test_http_route" } function test_grpc_route() { @@ -232,6 +248,8 @@ function test_grpc_route() { echo "-----------------------------------------------------------------------------------------------------" unexpose_envoy_gateway_svc || true kubectl delete -f "${GATEWAY_API_DIR}/grpc-debug.yaml" + + log_time "test_grpc_route" } function test_tcp_route() { @@ -266,4 +284,6 @@ function test_tcp_route() { rm deleteme.txt unexpose_envoy_gateway_svc || true kubectl delete -f "${GATEWAY_API_DIR}/tcp-debug.yaml" + + log_time "test_tcp_route" } diff --git a/dev/scripts/main.sh b/dev/scripts/main.sh index 6191b4a64..a7ee713a2 100644 --- a/dev/scripts/main.sh +++ b/dev/scripts/main.sh @@ -8,16 +8,19 @@ function setup_cluster() { [[ -z "${NAMESPACE}" ]] && echo "ERROR: [setup_cluster] Namespace name is required" && return 1 echo "Cluster name: ${CLUSTER_NAME}" - local count=$(kind get clusters -q | grep -c -sw "${CLUSTER_NAME}") + local count + + count=$(kind get clusters -q | grep -c -sw "${CLUSTER_NAME}") if [[ $count -eq 0 ]]; then echo "Cluster '${CLUSTER_NAME}' not found" - kind create cluster -n "${CLUSTER_NAME}" - kubectl create ns "${NAMESPACE}" + kind create cluster -n "${CLUSTER_NAME}" --config="${CUR_DIR}/../dev-cluster.yaml" else echo "Cluster '${CLUSTER_NAME}' found" fi - setup_kubectl_context + setup_kubectl_context + + log_time "setup_cluster" } function destroy_cluster() { @@ -53,6 +56,8 @@ function deploy_fullstack_cluster_setup_chart() { kubectl get clusterrole "${POD_MONITOR_ROLE}" -o wide kubectl get gatewayclass echo "" + + log_time "deploy_fullstack_cluster_setup_chart" } function destroy_fullstack_cluster_setup_chart() { @@ -72,6 +77,8 @@ function destroy_fullstack_cluster_setup_chart() { kubectl get clusterrole "${POD_MONITOR_ROLE}" -o wide kubectl get gatewayclass echo "" + + log_time "destroy_fullstack_cluster_setup_chart" } function install_chart() { @@ -83,7 +90,7 @@ function install_chart() { echo "" echo "Installing helm chart... " echo "SCRIPT_NAME: ${node_setup_script}" - echo "Values: -f ${CHART_DIR}/values.yaml --values ${CHART_VALUES_FILES}" + echo "Additional values: ${CHART_VALUES_FILES}" echo "-----------------------------------------------------------------------------------------------------" local count=$(helm list -q -n "${NAMESPACE}" | grep -c "${HELM_RELEASE_NAME}") if [[ $count -eq 0 ]]; then @@ -95,6 +102,8 @@ function install_chart() { else echo "${HELM_RELEASE_NAME} is already installed" fi + + log_time "install_chart" } function uninstall_chart() { @@ -111,6 +120,10 @@ function uninstall_chart() { else echo "Helm chart '${HELM_RELEASE_NAME}' not found in namespace ${NAMESPACE}. Nothing to uninstall. " fi + + kubectl delete ns "${NAMESPACE}" || true + + log_time "uninstall_chart" } function nmt_install() { @@ -161,4 +174,6 @@ function run_helm_chart_tests() { echo "Returning exit code 1" return 1 fi + + log_time "run_helm_chart_tests" } \ No newline at end of file diff --git a/dev/scripts/telemetry.sh b/dev/scripts/telemetry.sh index 15a258300..310be5d24 100644 --- a/dev/scripts/telemetry.sh +++ b/dev/scripts/telemetry.sh @@ -18,6 +18,8 @@ function fetch-prometheus-operator-bundle() { [[ "${status}" != 0 ]] && rm "${PROMETHEUS_OPERATOR_YAML}" && echo "ERROR: Failed to fetch prometheus bundle" return "${status}" fi + + log_time "fetch-prometheus-operator-bundle" } function deploy-prometheus-operator() { @@ -36,6 +38,8 @@ function deploy-prometheus-operator() { echo "Prometheus operator CRD is already installed" echo "" fi + + log_time "deploy-prometheus-operator" } function destroy-prometheus-operator() { @@ -45,6 +49,8 @@ function destroy-prometheus-operator() { echo "-----------------------------------------------------------------------------------------------------" kubectl delete -f "${PROMETHEUS_OPERATOR_YAML}" sleep 10 + + log_time "destroy-prometheus-operator" } function deploy-prometheus() { @@ -77,6 +83,8 @@ function deploy-prometheus() { echo "Waiting for prometheus to be active (timeout 300s)..." kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=prometheus --timeout 300s -n "${NAMESPACE}" + + log_time "deploy-prometheus" } function destroy-prometheus() { @@ -89,6 +97,8 @@ function destroy-prometheus() { kubectl delete -f "${PROMETHEUS_RBAC_YAML}" || true kubectl delete clusterrolebindings prometheus || true sleep 5 + + log_time "destroy-prometheus" } function deploy-prometheus-example-app() { @@ -143,6 +153,8 @@ function deploy_grafana_tempo() { helm upgrade -f "${TELEMETRY_DIR}/grafana/grafana-values.yaml" --install grafana grafana/grafana echo "Waiting for grafana to be active (timeout 300s)..." kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana" --timeout=300s + + log_time "deploy_grafana_tempo" } function destroy_grafana_tempo() {