Skip to content

Commit

Permalink
[ci]Optimization of ci runtime (#211)
Browse files Browse the repository at this point in the history 
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
  • Loading branch information
@zhuwenxing
zhuwenxing authored Oct 8, 2023
1 parent 1f8d535 commit af4a02c
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 38 deletions.
100 changes: 62 additions & 38 deletions .github/workflows/main.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ on:
- 'deployment/**'
- '**/*.go'
- '**/go.mod'
- '**/go.sum'
- '**/go.sum'
branches:
- main
workflow_dispatch:
Expand All @@ -53,7 +53,7 @@ jobs:
milvus_mode: [standalone]
another_milvus_mode: [standalone]
# mq_type: [pulsar, kafka] # TODO: add pulsar and kafka

steps:
- uses: actions/checkout@v3

Expand All @@ -67,23 +67,23 @@ jobs:
with:
go-version: '1.18.0'
cache: true

- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency

timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy

timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
Expand All @@ -92,13 +92,13 @@ jobs:
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Backup
timeout-minutes: 5
shell: bash
Expand All @@ -116,14 +116,14 @@ jobs:
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
python example/verify_data.py
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
Expand All @@ -138,7 +138,7 @@ jobs:
run: |
docker-compose down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
Expand All @@ -148,14 +148,14 @@ jobs:
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
Expand All @@ -174,7 +174,7 @@ jobs:
shell: bash
run: |
python example/verify_data.py
test-backup-restore-cli:
runs-on: ubuntu-latest
strategy:
Expand All @@ -184,7 +184,7 @@ jobs:
milvus_mode: [standalone, cluster]
another_milvus_mode: [standalone, cluster]
# mq_type: [pulsar, kafka] # TODO: add pulsar and kafka

steps:
- uses: actions/checkout@v3

Expand All @@ -198,37 +198,37 @@ jobs:
with:
go-version: '1.18.0'
cache: true

- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency

timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy

timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Backup
timeout-minutes: 5
shell: bash
Expand All @@ -246,14 +246,14 @@ jobs:
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
python example/verify_data.py
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
Expand All @@ -268,7 +268,7 @@ jobs:
run: |
docker-compose down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
Expand All @@ -277,14 +277,14 @@ jobs:
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
Expand All @@ -302,7 +302,7 @@ jobs:
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
python example/verify_data.py
test-backup-restore-with-custom-config:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -347,7 +347,7 @@ jobs:
yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml
yq -i '.minio.rootPath = "${{ matrix.milvus_minio_rootpath }}"' configs/backup.yaml
yq -i '.minio.backupBucketName = "${{ matrix.backup_bucket_name }}"' configs/backup.yaml
fi
yq -i '.log.level = "debug"' configs/backup.yaml
cat configs/backup.yaml || true
Expand Down Expand Up @@ -379,7 +379,7 @@ jobs:
sleep 10
kubectl get pods -n default | grep milvus-backup
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
docker-compose up -d
bash ../../scripts/check_healthy.sh
Expand Down Expand Up @@ -408,13 +408,37 @@ jobs:
shell: bash
run: |
python example/verify_data.py
- name: Export logs
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
bash ../../scripts/export_log_k8s.sh default milvus-backup logs
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v2
with:
name: custom-config-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
path: |
./logs
./server.log
/tmp/ci_logs
deployment/${{ matrix.milvus_mode }}/logs
test-backup-restore-api:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose, helm]
milvus_mode: [standalone, cluster]
deploy_tools: [docker-compose]
milvus_mode: [standalone]
case_tag: [L0, L1]
exclude:
- deploy_tools: helm
milvus_mode: cluster
Expand Down Expand Up @@ -478,7 +502,7 @@ jobs:
sleep 10
kubectl get pods -n default | grep milvus-backup
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
docker-compose up -d
bash ../../scripts/check_healthy.sh
Expand All @@ -496,15 +520,15 @@ jobs:
shell: bash
working-directory: tests
run: |
pytest -s -v --log-cli-level=INFO --capture=no --tags L0 L1
pytest -s -v --log-cli-level=INFO --capture=no --tags ${{ matrix.case_tag }}
- name: Get Milvus status
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose ps -a || true
kubectl get pods -n default || true
- name: Export logs
if: ${{ always() }}
shell: bash
Expand All @@ -513,16 +537,16 @@ jobs:
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
bash ../../scripts/export_log_k8s.sh default milvus-backup logs
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
fi
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v2
with:
name: logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
name: api-test-logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
path: |
./logs
./server.log
Expand Down
3 changes: 3 additions & 0 deletions tests/testcases/test_restore_backup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def test_milvus_restore_back(self, collection_type, collection_need_to_restore,
for name in restore_collections:
self.compare_collections(name, name+suffix)

@pytest.mark.tags(CaseLabel.L1)
def test_milvus_restore_back_with_db_support(self):
# prepare data
self._connect()
Expand Down Expand Up @@ -128,6 +129,7 @@ def test_milvus_restore_back_with_db_support(self):
@pytest.mark.parametrize("include_partition_key", [True, False])
@pytest.mark.parametrize("include_dynamic", [True, False])
@pytest.mark.parametrize("include_json", [True, False])
@pytest.mark.tags(CaseLabel.L1)
def test_milvus_restore_back_with_new_feature_support(self, include_json, include_dynamic, include_partition_key):
self._connect()
name_origin = cf.gen_unique_str(prefix)
Expand Down Expand Up @@ -201,6 +203,7 @@ def test_milvus_restore_back_with_new_feature_support(self, include_json, includ

@pytest.mark.parametrize("drop_db", [True, False])
@pytest.mark.parametrize("str_json", [True, False])
@pytest.mark.tags(CaseLabel.L1)
def test_milvus_restore_with_db_collections(self, drop_db, str_json):
# prepare data
self._connect()
Expand Down

0 comments on commit af4a02c

Please sign in to comment.