Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci]Optimization of ci runtime #211

Merged
merged 1 commit into from
Oct 8, 2023
Merged

[ci]Optimization of ci runtime #211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 62 additions & 38 deletions .github/workflows/main.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ on:
- 'deployment/**'
- '**/*.go'
- '**/go.mod'
- '**/go.sum'
- '**/go.sum'
branches:
- main
workflow_dispatch:
Expand All @@ -53,7 +53,7 @@ jobs:
milvus_mode: [standalone]
another_milvus_mode: [standalone]
# mq_type: [pulsar, kafka] # TODO: add pulsar and kafka

steps:
- uses: actions/checkout@v3

Expand All @@ -67,23 +67,23 @@ jobs:
with:
go-version: '1.18.0'
cache: true

- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency

timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy

timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
Expand All @@ -92,13 +92,13 @@ jobs:
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Backup
timeout-minutes: 5
shell: bash
Expand All @@ -116,14 +116,14 @@ jobs:
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
python example/verify_data.py
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
Expand All @@ -138,7 +138,7 @@ jobs:
run: |
docker-compose down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
Expand All @@ -148,14 +148,14 @@ jobs:
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
Expand All @@ -174,7 +174,7 @@ jobs:
shell: bash
run: |
python example/verify_data.py
test-backup-restore-cli:
runs-on: ubuntu-latest
strategy:
Expand All @@ -184,7 +184,7 @@ jobs:
milvus_mode: [standalone, cluster]
another_milvus_mode: [standalone, cluster]
# mq_type: [pulsar, kafka] # TODO: add pulsar and kafka

steps:
- uses: actions/checkout@v3

Expand All @@ -198,37 +198,37 @@ jobs:
with:
go-version: '1.18.0'
cache: true

- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency

timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy

timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Backup
timeout-minutes: 5
shell: bash
Expand All @@ -246,14 +246,14 @@ jobs:
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
python example/verify_data.py
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
Expand All @@ -268,7 +268,7 @@ jobs:
run: |
docker-compose down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
Expand All @@ -277,14 +277,14 @@ jobs:
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
Expand All @@ -302,7 +302,7 @@ jobs:
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
python example/verify_data.py
test-backup-restore-with-custom-config:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -347,7 +347,7 @@ jobs:
yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml
yq -i '.minio.rootPath = "${{ matrix.milvus_minio_rootpath }}"' configs/backup.yaml
yq -i '.minio.backupBucketName = "${{ matrix.backup_bucket_name }}"' configs/backup.yaml
fi
yq -i '.log.level = "debug"' configs/backup.yaml
cat configs/backup.yaml || true
Expand Down Expand Up @@ -379,7 +379,7 @@ jobs:
sleep 10
kubectl get pods -n default | grep milvus-backup
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
docker-compose up -d
bash ../../scripts/check_healthy.sh
Expand Down Expand Up @@ -408,13 +408,37 @@ jobs:
shell: bash
run: |
python example/verify_data.py
- name: Export logs
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
bash ../../scripts/export_log_k8s.sh default milvus-backup logs
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v2
with:
name: custom-config-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
path: |
./logs
./server.log
/tmp/ci_logs
deployment/${{ matrix.milvus_mode }}/logs
test-backup-restore-api:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose, helm]
milvus_mode: [standalone, cluster]
deploy_tools: [docker-compose]
milvus_mode: [standalone]
case_tag: [L0, L1]
exclude:
- deploy_tools: helm
milvus_mode: cluster
Expand Down Expand Up @@ -478,7 +502,7 @@ jobs:
sleep 10
kubectl get pods -n default | grep milvus-backup
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
docker-compose up -d
bash ../../scripts/check_healthy.sh
Expand All @@ -496,15 +520,15 @@ jobs:
shell: bash
working-directory: tests
run: |
pytest -s -v --log-cli-level=INFO --capture=no --tags L0 L1
pytest -s -v --log-cli-level=INFO --capture=no --tags ${{ matrix.case_tag }}
- name: Get Milvus status
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose ps -a || true
kubectl get pods -n default || true
- name: Export logs
if: ${{ always() }}
shell: bash
Expand All @@ -513,16 +537,16 @@ jobs:
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
bash ../../scripts/export_log_k8s.sh default milvus-backup logs
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
fi
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v2
with:
name: logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
name: api-test-logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
path: |
./logs
./server.log
Expand Down
3 changes: 3 additions & 0 deletions tests/testcases/test_restore_backup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def test_milvus_restore_back(self, collection_type, collection_need_to_restore,
for name in restore_collections:
self.compare_collections(name, name+suffix)

@pytest.mark.tags(CaseLabel.L1)
def test_milvus_restore_back_with_db_support(self):
# prepare data
self._connect()
Expand Down Expand Up @@ -128,6 +129,7 @@ def test_milvus_restore_back_with_db_support(self):
@pytest.mark.parametrize("include_partition_key", [True, False])
@pytest.mark.parametrize("include_dynamic", [True, False])
@pytest.mark.parametrize("include_json", [True, False])
@pytest.mark.tags(CaseLabel.L1)
def test_milvus_restore_back_with_new_feature_support(self, include_json, include_dynamic, include_partition_key):
self._connect()
name_origin = cf.gen_unique_str(prefix)
Expand Down Expand Up @@ -201,6 +203,7 @@ def test_milvus_restore_back_with_new_feature_support(self, include_json, includ

@pytest.mark.parametrize("drop_db", [True, False])
@pytest.mark.parametrize("str_json", [True, False])
@pytest.mark.tags(CaseLabel.L1)
def test_milvus_restore_with_db_collections(self, drop_db, str_json):
# prepare data
self._connect()
Expand Down
Loading