[ci]Refine ci (zilliztech#221)

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

.github/workflows/main.yaml

@@ -520,7 +520,7 @@ jobs:
         shell: bash
         working-directory: tests
         run: |
-          pytest -s -v --log-cli-level=INFO --capture=no --tags ${{ matrix.case_tag }}
+          pytest -s -v --tags ${{ matrix.case_tag }} -n 4
 
       - name: Get Milvus status
         shell: bash

.github/workflows/nightly.yaml

@@ -1,4 +1,4 @@
-name: Test
+name: Nightly Test
 
 on:
   workflow_dispatch:
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        deploy_tools: [helm]
+        deploy_tools: [docker-compose]
         milvus_mode: [standalone]
         milvus_version: [master-latest, 2.3.0-latest, 2.2.0-latest]
 
@@ -75,6 +75,8 @@ jobs:
           fi
 
           if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            yq -i '.services.standalone.image= "milvusdb/milvus:${{ matrix.milvus_version }}"' docker-compose.yml
+            cat docker-compose.yml
             docker-compose up -d
             bash ../../scripts/check_healthy.sh
             docker-compose ps -a
@@ -91,7 +93,7 @@ jobs:
         shell: bash
         working-directory: tests
         run: |
-          pytest -s -v --log-cli-level=INFO --capture=no
+          pytest -s -v --tags L0, L1, L2, L3
 
       - name: Get Milvus status
         shell: bash

.github/workflows/perf.yaml

@@ -0,0 +1,125 @@
+name: Perf Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  test-backup-restore-api:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        deploy_tools: [docker-compose]
+        milvus_mode: [standalone]
+        milvus_version: [master-latest, 2.3.0-latest, 2.2.0-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+          cache: pip
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18.0'
+          cache: true
+
+      - name: Creating kind cluster
+        if: ${{ matrix.deploy_tools == 'helm' }}
+        uses: helm/kind-action@v1.2.0
+
+      - name: Build
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          if [ ${{ matrix.deploy_tools }} == 'helm' ]; then
+            yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml
+            yq -i '.minio.backupBucketName = "milvus-bucket"' configs/backup.yaml
+            yq -i '.minio.rootPath = "file"' configs/backup.yaml
+          fi
+          yq -i '.log.level = "debug"' configs/backup.yaml
+          cat configs/backup.yaml || true
+          go get
+          go build
+      - name: Install dependency
+
+        timeout-minutes: 5
+        working-directory: tests
+        shell: bash
+        run: |
+          pip install -r requirements.txt --trusted-host https://test.pypi.org
+
+      - name: Milvus deploy
+
+        timeout-minutes: 15
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools }} == "helm" ]; then
+            helm repo add milvus https://milvus-io.github.io/milvus-helm
+            helm repo update
+            helm install --wait --timeout 600s milvus-backup milvus/milvus --set image.all.tag=${{ matrix.milvus_version }} -f values.yaml
+            kubectl get pods
+            kubectl port-forward service/milvus-backup 19530 >/dev/null 2>&1 &
+            kubectl port-forward service/milvus-backup-minio 9000  >/dev/null 2>&1 &
+            sleep 10
+            nc -vz 127.0.0.1 19530
+            nc -vz 127.0.0.1 9000
+            sleep 10
+            kubectl get pods -n default | grep milvus-backup
+          fi
+
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            docker-compose up -d
+            bash ../../scripts/check_healthy.sh
+            docker-compose ps -a
+          fi
+
+      - name: Start server
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          ./milvus-backup server > server.log 2>&1 &
+
+      - name: Run test
+        timeout-minutes: 120
+        shell: bash
+        working-directory: tests
+        run: |
+          pytest -s -v --tags Perf
+
+      - name: Get Milvus status
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          docker-compose ps -a || true
+          kubectl get pods -n default || true
+
+      - name: Export logs
+        if: ${{ always() }}
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools}} == "helm" ]; then
+            bash ../../scripts/export_log_k8s.sh default milvus-backup logs
+          fi
+
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            bash ../../scripts/export_log_docker.sh logs
+          fi
+
+      - name: Upload logs
+        if: ${{ ! success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
+          path: |
+            ./logs
+            ./server.log
+            /tmp/ci_logs
+            deployment/${{ matrix.milvus_mode }}/logs

tests/common/common_type.py

@@ -242,5 +242,4 @@ class CaseLabel:
     L1 = "L1"
     L2 = "L2"
     L3 = "L3"
-    Loadbalance = "Loadbalance"  # loadbalance testcases which need to be run in multi query nodes
-    ClusterOnly = "ClusterOnly"  # For functions only suitable to cluster mode
+    Perf = "Perf"

tests/testcases/test_backup_e2e.py

@@ -17,7 +17,7 @@
 class TestE2e(TestcaseBase):
     """ Test case of end to end"""
 
-    @pytest.mark.tags(CaseLabel.L1)
+    @pytest.mark.tags(CaseLabel.L0)
     def test_milvus_backup_default(self):
         # prepare data
         name_origin = cf.gen_unique_str(c_name_prefix)

tests/testcases/test_backup_perf.py

@@ -3,7 +3,6 @@
 
 from base.client_base import TestcaseBase
 from common import common_func as cf
-from common import common_type as ct
 from common.common_type import CaseLabel
 from utils.util_log import test_log as log
 from api.milvus_backup import MilvusBackupClient
@@ -14,18 +13,21 @@
 client = MilvusBackupClient("http://localhost:8080/api/v1")
 
 
+@pytest.mark.tags(CaseLabel.Perf)
 class TestPerf(TestcaseBase):
     """ Test case of performance"""
 
-    @pytest.mark.tags(CaseLabel.L3)
     def test_milvus_create_backup_perf(self):
         # prepare data
-        total_nb = 1000000
+        total_nb = 10000
         cnt = 10
-        collection_to_backup = cf.gen_unique_str(c_name_prefix)
-        for i in range(cnt):
-            self.prepare_data(collection_to_backup, nb=total_nb // cnt)
-        collections_to_backup = [collection_to_backup]
+        coll_num = 2
+        collections_to_backup = []
+        for i in range(coll_num):
+            collection_to_backup = cf.gen_unique_str(c_name_prefix)
+            for j in range(cnt):
+                self.prepare_data(collection_to_backup, nb=total_nb // cnt)
+            collections_to_backup.append(collection_to_backup)
         checkers = {
             Op.create: BackupCreateChecker(collections_to_backup)
         }
@@ -39,15 +41,17 @@ def test_milvus_create_backup_perf(self):
         sleep(10)
         log.info("*********************Perf Test End**********************")
 
-    @pytest.mark.tags(CaseLabel.L3)
     def test_milvus_restore_backup_perf(self):
         # prepare data
-        total_nb = 1000000
+        total_nb = 10000
         cnt = 10
-        collection_to_backup = cf.gen_unique_str(c_name_prefix)
-        for i in range(cnt):
-            self.prepare_data(collection_to_backup, nb=total_nb // cnt)
-        collections_to_backup = [collection_to_backup]
+        coll_num = 2
+        collections_to_backup = []
+        for i in range(coll_num):
+            collection_to_backup = cf.gen_unique_str(c_name_prefix)
+            for j in range(cnt):
+                self.prepare_data(collection_to_backup, nb=total_nb // cnt)
+            collections_to_backup.append(collection_to_backup)
         backup_name = cf.gen_unique_str(backup_prefix)
         suffix = "_bak"