Ronaldo

.github/workflows/nightly.yaml

@@ -0,0 +1,127 @@
+name: Nightly Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  test-backup-restore-api:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        deploy_tools: [docker-compose]
+        milvus_mode: [standalone]
+        milvus_version: [master-latest, 2.3.0-latest, 2.2.0-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+          cache: pip
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18.0'
+          cache: true
+
+      - name: Creating kind cluster
+        if: ${{ matrix.deploy_tools == 'helm' }}
+        uses: helm/kind-action@v1.2.0
+
+      - name: Build
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          if [ ${{ matrix.deploy_tools }} == 'helm' ]; then
+            yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml
+            yq -i '.minio.backupBucketName = "milvus-bucket"' configs/backup.yaml
+            yq -i '.minio.rootPath = "file"' configs/backup.yaml
+          fi
+          yq -i '.log.level = "debug"' configs/backup.yaml
+          cat configs/backup.yaml || true
+          go get
+          go build
+      - name: Install dependency
+
+        timeout-minutes: 5
+        working-directory: tests
+        shell: bash
+        run: |
+          pip install -r requirements.txt --trusted-host https://test.pypi.org
+
+      - name: Milvus deploy
+
+        timeout-minutes: 15
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools }} == "helm" ]; then
+            helm repo add milvus https://milvus-io.github.io/milvus-helm
+            helm repo update
+            helm install --wait --timeout 600s milvus-backup milvus/milvus --set image.all.tag=${{ matrix.milvus_version }} -f values.yaml
+            kubectl get pods
+            kubectl port-forward service/milvus-backup 19530 >/dev/null 2>&1 &
+            kubectl port-forward service/milvus-backup-minio 9000  >/dev/null 2>&1 &
+            sleep 10
+            nc -vz 127.0.0.1 19530
+            nc -vz 127.0.0.1 9000
+            sleep 10
+            kubectl get pods -n default | grep milvus-backup
+          fi
+
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            yq -i '.services.standalone.image= "milvusdb/milvus:${{ matrix.milvus_version }}"' docker-compose.yml
+            cat docker-compose.yml
+            docker-compose up -d
+            bash ../../scripts/check_healthy.sh
+            docker-compose ps -a
+          fi
+
+      - name: Start server
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          ./milvus-backup server > server.log 2>&1 &
+
+      - name: Run test
+        timeout-minutes: 120
+        shell: bash
+        working-directory: tests
+        run: |
+          pytest -s -v --tags L0, L1, L2, L3
+
+      - name: Get Milvus status
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          docker-compose ps -a || true
+          kubectl get pods -n default || true
+
+      - name: Export logs
+        if: ${{ always() }}
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools}} == "helm" ]; then
+            bash ../../scripts/export_log_k8s.sh default milvus-backup logs
+          fi
+
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            bash ../../scripts/export_log_docker.sh logs
+          fi
+
+      - name: Upload logs
+        if: ${{ ! success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
+          path: |
+            ./logs
+            ./server.log
+            /tmp/ci_logs
+            deployment/${{ matrix.milvus_mode }}/logs

.github/workflows/perf.yaml

@@ -0,0 +1,125 @@
+name: Perf Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  test-backup-restore-api:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        deploy_tools: [docker-compose]
+        milvus_mode: [standalone]
+        milvus_version: [master-latest, 2.3.0-latest, 2.2.0-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+          cache: pip
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18.0'
+          cache: true
+
+      - name: Creating kind cluster
+        if: ${{ matrix.deploy_tools == 'helm' }}
+        uses: helm/kind-action@v1.2.0
+
+      - name: Build
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          if [ ${{ matrix.deploy_tools }} == 'helm' ]; then
+            yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml
+            yq -i '.minio.backupBucketName = "milvus-bucket"' configs/backup.yaml
+            yq -i '.minio.rootPath = "file"' configs/backup.yaml
+          fi
+          yq -i '.log.level = "debug"' configs/backup.yaml
+          cat configs/backup.yaml || true
+          go get
+          go build
+      - name: Install dependency
+
+        timeout-minutes: 5
+        working-directory: tests
+        shell: bash
+        run: |
+          pip install -r requirements.txt --trusted-host https://test.pypi.org
+
+      - name: Milvus deploy
+
+        timeout-minutes: 15
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools }} == "helm" ]; then
+            helm repo add milvus https://milvus-io.github.io/milvus-helm
+            helm repo update
+            helm install --wait --timeout 600s milvus-backup milvus/milvus --set image.all.tag=${{ matrix.milvus_version }} -f values.yaml
+            kubectl get pods
+            kubectl port-forward service/milvus-backup 19530 >/dev/null 2>&1 &
+            kubectl port-forward service/milvus-backup-minio 9000  >/dev/null 2>&1 &
+            sleep 10
+            nc -vz 127.0.0.1 19530
+            nc -vz 127.0.0.1 9000
+            sleep 10
+            kubectl get pods -n default | grep milvus-backup
+          fi
+
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            docker-compose up -d
+            bash ../../scripts/check_healthy.sh
+            docker-compose ps -a
+          fi
+
+      - name: Start server
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          ./milvus-backup server > server.log 2>&1 &
+
+      - name: Run test
+        timeout-minutes: 120
+        shell: bash
+        working-directory: tests
+        run: |
+          pytest -s -v --tags Perf
+
+      - name: Get Milvus status
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          docker-compose ps -a || true
+          kubectl get pods -n default || true
+
+      - name: Export logs
+        if: ${{ always() }}
+        shell: bash
+        working-directory: deployment/${{ matrix.milvus_mode }}
+        run: |
+          if [ ${{ matrix.deploy_tools}} == "helm" ]; then
+            bash ../../scripts/export_log_k8s.sh default milvus-backup logs
+          fi
+
+          if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
+            bash ../../scripts/export_log_docker.sh logs
+          fi
+
+      - name: Upload logs
+        if: ${{ ! success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
+          path: |
+            ./logs
+            ./server.log
+            /tmp/ci_logs
+            deployment/${{ matrix.milvus_mode }}/logs

.github/workflows/release.yaml

@@ -28,4 +28,5 @@ jobs:
           version: latest
           args: release --clean
         env:
-          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}  
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+          GITHUB_HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }}

.goreleaser.yaml

@@ -10,6 +10,14 @@ builds:
     goos:
       - linux
       - darwin
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -w -s
+      - -X main.version={{ .Version }}
+      - -X main.commit={{ .Commit }}
+      - -X main.date={{ .Date }}
 
 archives:
   - format: tar.gz
@@ -25,12 +33,26 @@ checksum:
   name_template: 'checksums.txt'
 snapshot:
   name_template: "{{ incpatch .Version }}-next"
+
+brews:
+  -
+    name: milvus-backup
+    folder: Formula
+    homepage: "https://github.com/zilliztech/milvus-backup"
+    repository:
+      owner: zilliztech
+      name: homebrew-tap
+      token: "{{ .Env.GITHUB_HOMEBREW_TOKEN }}"
+
+
 changelog:
   sort: asc
+  use: github
   filters:
     exclude:
       - '^docs:'
       - '^test:'
+      - Merge pull request
 
 # The lines beneath this are called `modelines`. See `:help modeline`
 # Feel free to remove those if you don't want/use them.

README.md

@@ -2,7 +2,17 @@
 
 Milvus-Backup is a tool that allows users to backup and restore Milvus data. This tool can be utilized either through the command line or an API server.
 
-In order to use Milvus-Backup effectively, access to Milvus proxy and Minio cluster is required. Configuration settings related to this access can be edited in `configs/backup.yaml`.
+The Milvus-backup process has negligible impact on the performance of Milvus. Milvus cluster is fully functional and can operate normally while backup and restoration are in progress.
+
+## Compatibility
+|      Milvus      |  Milvus-backup   |
+|:----------------:|:----------------:|
+| v2.3.0 and above | v0.4.0 and above | 
+| v2.2.9 and above | v0.3.0 and above |
+| v2.2.0 to v2.2.8 | v0.1.0 to v0.2.2 |
+
+## Config
+In order to use Milvus-Backup, access to Milvus proxy and Minio cluster is required. Configuration settings related to this access can be edited in `configs/backup.yaml`.
 
 > **Note**
 >
@@ -14,8 +24,6 @@ In order to use Milvus-Backup effectively, access to Milvus proxy and Minio clus
 > |bucketName|a-bucket|milvus-bucket|
 > |rootPath|files|file|
 
-The Milvus-backup process has negligible impact on the performance of Milvus. Milvus cluster is fully functional and can operate normally while backup and restoration are in progress.
-
 ## Development
 
 ### Build
@@ -137,14 +145,14 @@ curl --location --request GET 'http://localhost:8080/api/v1/get_restore?id=test_
 Milvus-backup establish CLI based on cobra. Use the following command to see the usage.
 
 ```
-milvus-backup is a backup tool for milvus.
+milvus-backup is a backup&restore tool for milvus.
 
 Usage:
   milvus-backup [flags]
   milvus-backup [command]
 
 Available Commands:
-  completion  Generate the autocompletion script for the specified shell
+  check       check if the connects is right.
   create      create subcommand create a backup.
   delete      delete subcommand delete backup by name.
   get         get subcommand get backup by name.
@@ -154,7 +162,8 @@ Available Commands:
   server      server subcommand start milvus-backup RESTAPI server.
 
 Flags:
-  -h, --help   help for milvus-backup
+      --config string   config YAML file of milvus (default "backup.yaml")
+  -h, --help            help for milvus-backup
 
 Use "milvus-backup [command] --help" for more information about a command.
 ```
@@ -163,6 +172,26 @@ Use "milvus-backup [command] --help" for more information about a command.
 
 To try this demo, you should have a functional Milvus server installed and have pymilvus library installed.
 
+Step 0: Check the connections
+
+First of all, we can use `check` command to check whether connections to milvus and storage is normal:
+
+```
+./milvus-backup check
+```
+
+normal output:
+
+```shell
+Succeed to connect to milvus and storage.
+Milvus version: v2.3
+Storage:
+milvus-bucket: a-bucket
+milvus-rootpath: files
+backup-bucket: a-bucket
+backup-rootpath: backup
+```
+
 Step 1: Prepare the Data
 
 Create a collection in Milvus called `hello_milvus` and insert some data using the following command: