Add upgrade test for spiderpool

Signed-off-by: tao.yang <tao.yang@daocloud.io>

.github/workflows/auto-upgrade-ci.yaml

@@ -0,0 +1,287 @@
+name: Auto Upgrade Test
+
+permissions: write-all
+
+env:
+ CLUSTER_NAME: spider
+ E2E_TIME_OUT: 60m
+ KUBECONFIG_PATH: /home/runner/work/spiderpool/spiderpool/test/.cluster
+on:
+ schedule:
+ - cron: "0 20 * * *"
+
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: 'sha, tag, branch. also this is the version you will upgrade to.'
+ required: true
+ default: main
+ e2e_enabled:
+ description: 'run e2e test'
+ required: false
+ type: choice
+ default: "false"
+ options:
+ - "true"
+ - "false"
+
+jobs:
+ get_ref:
+ runs-on: ubuntu-latest
+ outputs:
+ ref: ${{ env.RUN_REF }}
+ e2e_enabled: ${{ env.RUN_E2E_ENABLED }}
+ steps:
+ - name: Get Ref
+ id: get_ref
+ run: |
+ if ${{ github.event_name == 'workflow_dispatch' }} ; then
+ echo "call by self workflow_dispatch"
+ echo "RUN_TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
+ if ${{ github.event.inputs.e2e_enabled == 'true' }}; then
+ echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
+ else
+ echo "RUN_E2E_ENABLED=false" >> $GITHUB_ENV
+ fi
+ else
+ # schedule event
+ # use main sha for ci image tag
+ echo "trigger by schedule"
+ echo "RUN_TAG=main" >> $GITHUB_ENV
+ echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
+ echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV
+ fi
+
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ ref: ${{ env.RUN_TAG }}
+
+ - name: Result Ref
+ id: result
+ run: |
+ ref=$( git show -s --format='format:%H')
+ echo "RUN_REF=${ref}" >> $GITHUB_ENV
+
+ call_build_ci_image:
+ needs: [get_ref]
+ uses: ./.github/workflows/build-image-ci.yaml
+ with:
+ ref: ${{ needs.get_ref.outputs.ref }}
+ push: false
+ secrets: inherit
+
+ lint_chart_against_release_image:
+ needs: get_ref
+ uses: ./.github/workflows/call-lint-chart.yaml
+ with:
+ ref: ${{ needs.get_ref.outputs.ref }}
+ secrets: inherit
+
+ trivy_scan_images:
+ needs: [call_build_ci_image, get_ref]
+ uses: ./.github/workflows/trivy-scan-image.yaml
+ with:
+ image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }}
+ ref: ${{ needs.get_ref.outputs.ref }}
+ secrets: inherit
+
+ run_upgrade_tests_on_release_version:
+ needs: [call_build_ci_image, get_ref]
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ # Synchronise with the latest releases of each version
+ version: [release-v0.7, release-v0.8]
+ steps:
+ - name: Free Disk Space (Ubuntu)
+ uses: jlumbroso/free-disk-space@main
+ with:
+ # this might remove tools that are actually needed,
+ # if set to "true" but frees about 6 GB
+ tool-cache: false
+ # all of these default to true, but feel free to set to
+ # "false" if necessary for your workflow
+ android: true
+ dotnet: true
+ haskell: true
+ large-packages: true
+ docker-images: true
+ swap-storage: true
+
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ ref: ${{ matrix.version }}
+
+ - name: Get image tag
+ run: |
+ cat VERSION
+ imageTag=$(grep -E "^[[:space:]]*v[0-9]+.[0-9]+.[0-9]+[[:space:]]*$" VERSION)
+ echo "imageTag=$imageTag" >> $GITHUB_ENV
+
+ - name: Prepare
+ id: prepare
+ run: |
+ echo "ref: ${{ inputs.ref }} "
+ echo "===== image "
+ echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ env.imageTag }}"
+ TMP=` date +%m%d%H%M%S `
+ E2E_CLUSTER_NAME="spiderpool${TMP}"
+ echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV
+
+ - name: Setup Golang
+ uses: actions/setup-go@v5
+ with:
+ go-version: 1.21.4
+
+ # https://github.com/helm/kind-action
+ - name: Install Kind Bin
+ uses: helm/kind-action@v1.8.0
+ with:
+ install_only: true
+
+ - name: Install Tools
+ run: |
+ # install kind/p2ctl/helm/ginkgo
+ bash ./test/scripts/install-tools.sh
+
+ # test against commit version
+ # https://github.com/kubernetes-sigs/kind/issues/2863
+ - name: Setup Kind Cluster on ${{ matrix.version }}
+ uses: nick-invision/retry@v2
+ with:
+ timeout_minutes: 20
+ max_attempts: 3
+ command: |
+ make e2e_init_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \
+ -e E2E_SPIDERPOOL_TAG=${{ env.imageTag }} \
+ -e PYROSCOPE_LOCAL_PORT=""
+
+ - name: Move kubeconfig
+ run: |
+ cp -r ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config /tmp
+
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ ref: ${{ needs.get_ref.outputs.ref }}
+ fetch-depth: 0
+
+ - name: Move kubeconfig to branch ${{ needs.get_ref.outputs.ref }}
+ run: |
+ mkdir -p ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/
+ cp -r /tmp/config ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config
+
+ - name: Download spiderpool-agent image
+ uses: actions/download-artifact@v3
+ with:
+ name: image-tar-spiderpool-agent
+ path: test/.download
+
+ - name: Download spiderpool-controller image
+ uses: actions/download-artifact@v3
+ with:
+ name: image-tar-spiderpool-controller
+ path: test/.download
+
+ - name: Load Images
+ run: |
+ TAR_FILES=` ls test/.download `
+ echo $TAR_FILES
+ for ITEM in $TAR_FILES ; do
+ IMAGE_NAME=${ITEM%*.tar}
+ echo ${IMAGE_NAME}
+ cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }}
+ done
+
+ - name: Upgrade to version ${{ needs.get_ref.outputs.ref }}
+ continue-on-error: true
+ run: |
+ make upgrade_e2e_spiderpool -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \
+ -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \
+ -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \
+
+ - name: Run e2e Test on ${{ needs.get_ref.outputs.ref }}
+ id: run_e2e
+ continue-on-error: true
+ if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
+ run: |
+ RESULT=0
+ export KUBECONFIG=${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config
+ kubectl get po -A | grep "spiderpool"
+ RESULT=$(kubectl get po -A | grep "spiderpool" | grep -Ev "Running|Completed" | wc -l )
+ if ((RESULT==0)) ; then
+ echo "debug"
+ kubectl get po -l app=test-pod -oyaml
+ NODE_LIST=docker ps | grep "spiderpool" | awk '{print $1}'
+ for NODE in $NODE_LIST ; do
+ docker exec -ti $NODE /bin/bash rm -rf /etc/cni/net.d/multus.d /etc/cni/net.d/00-multus.conf
+ done
+ kubectl delete po -l app.kubernetes.io/component=spiderpool-agent
+ kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-agent --timeout=300s pod -n kube-system
+ sleep 600
+ kubectl get network-attachment-definitions.k8s.cni.cncf.io -A
+ kubectl delete po -l app=test-pod
+ if kubectl wait --for=condition=ready -l app=test-pod --timeout=300s pod; then
+ echo "RUN_E2E_PASS=true" >> $GITHUB_ENV
+ exit 0
+ else
+ kubectl describe po -l app=test-pod || KIND_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} ./scripts/debugEnv.sh ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config "detail" ;
+ echo "RUN_E2E_PASS=false" >> $GITHUB_ENV
+ exit 1
+ fi
+ else
+ echo "RUN_E2E_PASS=false" >> $GITHUB_ENV
+ kubectl describe po -n kube-system -l app.kubernetes.io/instance=spiderpool
+ fi
+
+ - name: Upload e2e log
+ if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
+ uses: actions/upload-artifact@v3.1.3
+ with:
+ name: ${{ matrix.version }}-${{ needs.get_ref.outputs.ref }}-debuglog.txt
+ path: test/e2edebugLog.txt
+ retention-days: 7
+
+ - name: Upload e2e report
+ if: ${{ env.UPLOAD_E2E_REPORT == 'true' }}
+ uses: actions/upload-artifact@v3.1.3
+ with:
+ name: ${{ matrix.version }}-${{ needs.get_ref.outputs.ref }}-e2ereport.json
+ path: e2ereport.json
+ retention-days: 1
+
+ - name: Show e2e Result
+ if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
+ run: |
+ if ${{ env.RUN_E2E_PASS == 'true' }} ;then
+ exit 0
+ else
+ exit 1
+ fi
+
+ creat_issue:
+ runs-on: ubuntu-latest
+ needs: [run_upgrade_tests_on_release_version,trivy_scan_images]
+ if: ${{ always() && needs.run_upgrade_tests_on_release_version.result == 'failure' }}
+ steps:
+ - name: echo
+ run: |
+ echo ${{ github.repository }}
+ echo ${{ github.repository_owner }}
+ echo "TIMESTAMP=`date +%Y-%m-%d`" >> $GITHUB_ENV
+
+ - name: create an issue
+ uses: dacbd/create-issue-action@v1.2.1
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ title: "upgrade CI ${{ ENV.TIMESTAMP }}: Failed"
+ body: |
+ action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ labels: "kind/ci-bug"
+ assignees: "ty-dc"

Makefile

@@ -428,6 +428,10 @@ clean: clean_e2e
 clean_e2e_spiderpool:
  -$(QUIET) make -C test uninstall_spiderpool
 
+.PHONY: upgrade_e2e_spiderpool
+upgrade_e2e_spiderpool:
+ -$(QUIET) make -C test upgrade_spiderpool
+
 .PHONY: codegen
 codegen:
  @echo "Generate k8s SDK with code-generator."

test/Makefile

@@ -389,6 +389,31 @@ uninstall_spiderpool:
  @ make helm_uninstall_spiderpool
  @ make clean_spiderpool_crd
 
+.PHONY: upgrade_spiderpool
+upgrade_spiderpool:
+ @echo -e "\033[35m [upgrade spiderpool] \033[0m"
+ @make helm_upgrade_spiderpool
+
+.PHONY: helm_upgrade_spiderpool
+helm_upgrade_spiderpool:
+ @echo -e "\033[35m [helm upgrade spiderpool] \033[0m"
+ cd $(ROOT_DIR)/charts/spiderpool/crds ; \
+ ls | grep '\.yaml$$' | xargs -I {} kubectl apply -f {} --kubeconfig $(E2E_KUBECONFIG) ; \
+ kubectl delete po -n $(RELEASE_NAMESPACE) spiderpool-init --kubeconfig $(E2E_KUBECONFIG) || true ;\
+ HELM_OPTION="";\
+ HELM_OPTION+=" --set spiderpoolController.replicas=1 " ; \
+ HELM_OPTION+=" --set spiderpoolAgent.image.registry="" \
+ --set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \
+ --set spiderpoolAgent.image.tag=$(E2E_SPIDERPOOL_TAG) \
+ --set spiderpoolController.image.registry="" \
+ --set spiderpoolController.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \
+ --set spiderpoolController.image.tag=$(E2E_SPIDERPOOL_TAG) \
+ --set spiderpoolInit.image.registry="" \
+ --set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \
+ --set spiderpoolInit.image.tag=$(E2E_SPIDERPOOL_TAG) " \
+ helm --kubeconfig $(E2E_KUBECONFIG) upgrade $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool $${HELM_OPTION} \
+ -n $(RELEASE_NAMESPACE) --wait || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail"; exit 1; } ; \
+
 .PHONY: clean
 clean:
  @rm -rf $(CLUSTER_DIR)