From 11b79e818456eeaef9ff1331c586023260d2f4fb Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Wed, 17 Jul 2024 16:57:32 -0600 Subject: [PATCH 1/4] Improve hotfix test * Change cache action to save operator binaries * Change starting MinIO version to `RELEASE.2024-01-31T20-20-33Z` Signed-off-by: pjuarezd --- .github/workflows/kubernetes-tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/kubernetes-tests.yml b/.github/workflows/kubernetes-tests.yml index ceddd1b29a3..c944ff9499e 100644 --- a/.github/workflows/kubernetes-tests.yml +++ b/.github/workflows/kubernetes-tests.yml @@ -192,15 +192,15 @@ jobs: - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} - - uses: actions/cache@v3 - name: Operator Binary Cache + - name: Operator Binary + run: | + make operator + - uses: actions/cache/save@v3 + name: Save Operator Binary with: path: | ./minio-operator key: ${{ runner.os }}-binary-${{ github.run_id }} - - name: Operator Binary - run: | - make operator sidecar: timeout-minutes: 30 runs-on: ${{ matrix.os }} @@ -217,16 +217,16 @@ jobs: - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} - - uses: actions/cache@v3 - name: Operator Sidecar Binary Cache - with: - path: | - ./sidecar/minio-operator-sidecar - key: ${{ runner.os }}-sidecar-binary-${{ github.run_id }} - name: Operator Sidecar Binary working-directory: ./sidecar run: | make sidecar + - uses: actions/cache/save@v3 + name: Save Operator Sidecar Binary + with: + path: | + ./sidecar/minio-operator-sidecar + key: ${{ runner.os }}-sidecar-binary-${{ github.run_id }} test-tenant: timeout-minutes: 30 runs-on: ${{ matrix.os }} @@ -347,7 +347,7 @@ jobs: key: ${{ runner.os }}-sidecar-binary-${{ github.run_id }} - name: Tenant upgrade to hotfix version run: | - "${GITHUB_WORKSPACE}/testing/tenant-hotfix-update.sh" "quay.io/minio/minio:RELEASE.2024-02-04T22-36-13Z" "quay.io/minio/minio:RELEASE.2024-02-04T22-36-13Z.hotfix.c40028f8f" + "${GITHUB_WORKSPACE}/testing/tenant-hotfix-update.sh" "quay.io/minio/minio:RELEASE.2024-01-31T20-20-33Z" "quay.io/minio/minio:RELEASE.2024-02-04T22-36-13Z.hotfix.c40028f8f" test-kes: timeout-minutes: 30 runs-on: ${{ matrix.os }} From 3c54c43d77e327662488ec495eb0811a8e3b260b Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Wed, 17 Jul 2024 17:05:58 -0600 Subject: [PATCH 2/4] remove unused scripts Signed-off-by: pjuarezd --- tests/common.sh | 132 ------------------------------------ tests/start-tests-tenant.sh | 65 ------------------ 2 files changed, 197 deletions(-) delete mode 100755 tests/common.sh delete mode 100755 tests/start-tests-tenant.sh diff --git a/tests/common.sh b/tests/common.sh deleted file mode 100755 index b3344df47e8..00000000000 --- a/tests/common.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# Copyright (C) 2022, MinIO, Inc. -# -# This code is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3, -# as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License, version 3, -# along with this program. If not, see - -source "${GITHUB_WORKSPACE}/shared-functions/shared-code.sh" # This is common.sh for k8s tests across multiple repos. - -yell() { echo "$0: $*" >&2; } - -die() { - yell "$*" - (kind delete cluster || true ) && exit 111 -} - -try() { "$@" || die "cannot $*"; } - -function setup_kind() { - # TODO once feature is added: https://github.com/kubernetes-sigs/kind/issues/1300 - { - printf "kind: Cluster\n" - printf "apiVersion: kind.x-k8s.io/v1alpha4\n" - printf "nodes:\n" - printf " - role: control-plane\n" - printf " - role: worker\n" - printf " - role: worker\n" - printf " - role: worker\n" - printf " - role: worker\n" - } >> kind-config.yaml - echo "---" - cat kind-config.yaml - echo "----" - try kind create cluster --config kind-config.yaml - echo "Kind is ready" - try kubectl get nodes -} - -function get_latest_release() { - curl --silent "https://api.github.com/repos/$1/releases/latest" | - grep '"tag_name":' | - sed -E 's/.*"([^"]+)".*/\1/' -} - -function install_operator() { - - OPR_LATEST=$(get_latest_release minio/operator) - echo " Load minio/operator image ($OPR_LATEST) to the cluster" - try kubectl apply -k "github.com/minio/operator/?ref=$OPR_LATEST" - echo "Waiting for k8s api" - sleep 10 - echo "Waiting for Operator Pods to come online (2m timeout)" - - try kubectl wait --namespace minio-operator \ - --for=condition=ready pod \ - --selector=name=minio-operator \ - --timeout=120s -} - -function destroy_kind() { - kind delete cluster -} - -function check_tenant_status() { - # Check MinIO is accessible - - waitdone=0 - totalwait=0 - while true; do - waitdone=$(kubectl -n $1 get pods -l v1.min.io/tenant=$2 --no-headers | wc -l) - if [ "$waitdone" -ne 0 ]; then - echo "Found $waitdone pods" - break - fi - sleep 5 - totalwait=$((totalwait + 5)) - if [ "$totalwait" -gt 305 ]; then - echo "Unable to create tenant after 5 minutes, exiting." - try false - fi - done - - echo "Waiting for pods to be ready. (5m timeout)" - - USER=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g') - PASSWORD=$(kubectl -n $1 get secrets $2-env-configuration -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g') - - try kubectl wait --namespace $1 \ - --for=condition=ready pod \ - --selector=v1.min.io/tenant=$2 \ - --timeout=300s - - echo "Tenant is created successfully, proceeding to validate 'mc admin info minio/'" - - kubectl run admin-mc -i --tty --image minio/mc --command -- bash -c "until (mc alias set minio/ https://minio.$1.svc.cluster.local $USER $PASSWORD); do echo \"...waiting... for 5secs\" && sleep 5; done; mc admin info minio/;" - - echo "Done." -} - -# Install tenant function is being used by deploy-tenant and check-prometheus -function install_tenant() { - - namespace=tenant-lite - key=v1.min.io/tenant - value=myminio - echo "Installing lite tenant" - - try kubectl apply -k "${GITHUB_WORKSPACE}/examples/kustomization/tenant-lite" - - echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled" - echo $namespace - echo $value - echo $key - wait_for_resource $namespace $value $key - - echo "Waiting for tenant pods to come online (5m timeout)" - try kubectl wait --namespace $namespace \ - --for=condition=ready pod \ - --selector $key=$value \ - --timeout=300s - - echo "Build passes basic tenant creation" - -} diff --git a/tests/start-tests-tenant.sh b/tests/start-tests-tenant.sh deleted file mode 100755 index 5f335cfd224..00000000000 --- a/tests/start-tests-tenant.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# Copyright (C) 2022, MinIO, Inc. -# -# This code is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3, -# as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License, version 3, -# along with this program. If not, see - -# This script requires: kubectl, kind - -SCRIPT_DIR=$(dirname "$0") -export SCRIPT_DIR - -source "${SCRIPT_DIR}/common.sh" - -function install_tenants() { - echo "Installing tenants" - - # Install lite & kes tenants - try kubectl apply -k "${SCRIPT_DIR}/../examples/kustomization/tenant-lite" - try kubectl apply -k "${SCRIPT_DIR}/../examples/kustomization/tenant-kes-encryption" - - echo "Waiting for the tenant statefulset, this indicates the tenant is being fulfilled" - waitdone=0 - totalwait=0 - while true; do - waitdone=$(kubectl -n tenant-lite get pods -l v1.min.io/tenant=myminio --no-headers | wc -l) - if [ "$waitdone" -ne 0 ]; then - echo "Found $waitdone pods" - break - fi - sleep 5 - totalwait=$((totalwait + 5)) - if [ "$totalwait" -gt 300 ]; then - echo "Tenant never created statefulset after 5 minutes" - try false - fi - done - - echo "Waiting for tenant pods to come online (5m timeout)" - try kubectl wait --namespace tenant-lite \ - --for=condition=ready pod \ - --selector="v1.min.io/tenant=myminio" \ - --timeout=300s - - echo "Build passes basic tenant creation" -} - -function main() { - destroy_kind - setup_kind - install_operator - install_tenants - check_tenant_status tenant-lite myminio - kubectl proxy & -} - -main "$@" From e4f2115bee9456f560fbd98e1e66e451d73535a2 Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Wed, 17 Jul 2024 17:16:03 -0600 Subject: [PATCH 3/4] remove debug lines and rely on sleeps, should evaluate operator deployment rollout instead Signed-off-by: pjuarezd --- testing/common.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/testing/common.sh b/testing/common.sh index ceae9f9d752..45b715527e2 100644 --- a/testing/common.sh +++ b/testing/common.sh @@ -572,13 +572,12 @@ function install_operator() { echo "Installing Current Operator with sts enabled" try kubectl apply -k "${SCRIPT_DIR}/../testing/sts/operator" try kubectl -n minio-operator set env deployment/minio-operator OPERATOR_SIDECAR_IMAGE="$SIDECAR_TAG" - echo "key, value for pod selector in kustomize test" + try kubectl -n minio-operator rollout status deployment/minio-operator key=name value=minio-operator elif [ "$1" = "certmanager" ]; then echo "Installing Current Operator with certmanager" try kubectl apply -k "${SCRIPT_DIR}/../testing/certmanager/operator" - echo "key, value for pod selector in kustomize test" key=name value=minio-operator else @@ -596,18 +595,12 @@ function install_operator() { # and then we change the images, no need to have more overlays in different folders. try kubectl -n minio-operator set image deployment/minio-operator minio-operator="$TAG" try kubectl -n minio-operator set env deployment/minio-operator OPERATOR_SIDECAR_IMAGE="$SIDECAR_TAG" - try kubectl -n minio-operator rollout status deployment/minio-operator - echo "key, value for pod selector in kustomize test" key=name value=minio-operator fi - # Reusing the wait for both, Kustomize and Helm - echo "Waiting for k8s api" - sleep 10 - kubectl get ns kubectl -n minio-operator get deployments @@ -692,11 +685,8 @@ function check_tenant_status() { key=v1.min.io/tenant value=$2 if [ $# -ge 3 ]; then - echo "Third argument provided, then set key value" key=app value=$3 - else - echo "No third argument provided, using default key" fi wait_resource_status $1 Tenant $2 600 From 0569279c9cbe8bd2a5c057fd66ed126058820ccc Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Wed, 17 Jul 2024 18:51:45 -0600 Subject: [PATCH 4/4] hotfix test usually is not taking more than 5 minutes, timing out in 10 minutes to fail fast Signed-off-by: pjuarezd --- .github/workflows/kubernetes-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kubernetes-tests.yml b/.github/workflows/kubernetes-tests.yml index c944ff9499e..399bca37593 100644 --- a/.github/workflows/kubernetes-tests.yml +++ b/.github/workflows/kubernetes-tests.yml @@ -319,7 +319,7 @@ jobs: # run: | # "${GITHUB_WORKSPACE}/testing/console-tenant+kes.sh" test-tenant-hotfix-update: - timeout-minutes: 30 + timeout-minutes: 10 runs-on: ${{ matrix.os }} needs: - operator