From a96d9861cf61e8f7b7cfa9087d57c2e4112c1e20 Mon Sep 17 00:00:00 2001 From: huanghaoyuanhhy Date: Tue, 12 Nov 2024 11:42:01 +0800 Subject: [PATCH 1/4] chore: lower log levels from info to debug Signed-off-by: huanghaoyuanhhy --- core/backup_impl_create_backup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index c26f74f..c433fed 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -777,10 +777,10 @@ func (b *BackupContext) executeCreateBackup(ctx context.Context, request *backup func (b *BackupContext) writeBackupInfoMeta(ctx context.Context, id string) error { backupInfo := b.meta.GetFullMeta(id) - log.Info("Final backupInfo", zap.String("backupInfo", backupInfo.String())) + log.Debug("Final backupInfo", zap.String("backupInfo", backupInfo.String())) output, _ := serialize(backupInfo) log.Debug("backup meta", zap.String("value", string(output.BackupMetaBytes))) - log.Info("collection meta", zap.String("value", string(output.CollectionMetaBytes))) + log.Debug("collection meta", zap.String("value", string(output.CollectionMetaBytes))) log.Debug("partition meta", zap.String("value", string(output.PartitionMetaBytes))) log.Debug("segment meta", zap.String("value", string(output.SegmentMetaBytes))) From 98f0526bcd2b844190e56072679f536a8ce12e29 Mon Sep 17 00:00:00 2001 From: wayblink Date: Fri, 22 Nov 2024 10:23:40 +0800 Subject: [PATCH 2/4] fix rbac miss role bug (#457) Signed-off-by: wayblink --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2c6b332..4d776c2 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20241030091852-d6eb85c1a8ff + require github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd require ( diff --git a/go.sum b/go.sum index a462d11..689b2ed 100644 --- a/go.sum +++ b/go.sum @@ -380,8 +380,6 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd h1:x0b0+foTe23sKcVFseR1DE8+BB08EH6ViiRHaz8PEik= github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16 h1:zsA/xt/HJ3HkWUgazbbkoIR8S5mcxAtypvoSKsemhMw= -github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16/go.mod h1:TdzShm5isV4F6kvrd+9V/CKowNIX+H+jvybF0WWts0Y= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= @@ -535,6 +533,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20241030091852-d6eb85c1a8ff h1:b73JFTui+bKEy5HuDFUOYNH3XNbYYl62Rv7JKA/thZo= +github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20241030091852-d6eb85c1a8ff/go.mod h1:TdzShm5isV4F6kvrd+9V/CKowNIX+H+jvybF0WWts0Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= From d837c4bb82f5885c110bc12501aa0b4855b98a4a Mon Sep 17 00:00:00 2001 From: Shehroz Ahmed Date: Thu, 28 Nov 2024 13:09:36 +0500 Subject: [PATCH 3/4] Update README.md (#469) Updated README.md file to add an example of how to sync minio to AWS S3 bucket and added a configuration table with all the allows configurations and their details. Signed-off-by: Shehroz Ahmed --- README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/README.md b/README.md index e39f063..87b0d62 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,51 @@ We offer access to our Swagger UI, which displays comprehensive information for ``` http://localhost:8080/api/v1/docs/index.html ``` +--- + +## Backup.yaml Configurations + +Below is a summary of the configurations supported in `backup.yaml`: + +| **Section** | **Field** | **Description** | **Default/Example** | +|--------------------|-----------------------------------|-------------------------------------------------------------------------------------------------------|-----------------------------------------------| +| `log` | `level` | Logging level. Supported: `debug`, `info`, `warn`, `error`, `panic`, `fatal`. | `info` | +| | `console` | Whether to print logs to the console. | `true` | +| | `file.rootPath` | Path to the log file. | `logs/backup.log` | +| `http` | `simpleResponse` | Whether to enable simple HTTP responses. | `true` | +| `milvus` | `address` | Milvus proxy address. | `localhost` | +| | `port` | Milvus proxy port. | `19530` | +| | `authorizationEnabled` | Whether to enable authorization. | `false` | +| | `tlsMode` | TLS mode (0: none, 1: one-way, 2: two-way). | `0` | +| | `user` | Username for Milvus. | `root` | +| | `password` | Password for Milvus. | `Milvus` | +| `minio` | `storageType` | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`). | `minio` | +| | `address` | MinIO/S3 address. | `localhost` | +| | `port` | MinIO/S3 port. | `9000` | +| | `accessKeyID` | MinIO/S3 access key ID. | `minioadmin` | +| | `secretAccessKey` | MinIO/S3 secret access key. | `minioadmin` | +| | `useSSL` | Whether to use SSL for MinIO/S3. | `false` | +| | `bucketName` | Bucket name in MinIO/S3. | `a-bucket` | +| | `rootPath` | Storage root path in MinIO/S3. | `files` | +| `minio (backup)` | `backupStorageType` | Backup storage type (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`). | `minio` | +| | `backupAddress` | Address of backup storage. | `localhost` | +| | `backupPort` | Port of backup storage. | `9000` | +| | `backupUseSSL` | Whether to use SSL for backup storage. | `false` | +| | `backupAccessKeyID` | Backup storage access key ID. | `minioadmin` | +| | `backupSecretAccessKey` | Backup storage secret access key. | `minioadmin` | +| | `backupBucketName` | Bucket name for backups. | `a-bucket` | +| | `backupRootPath` | Root path to store backup data. | `backup` | +| | `crossStorage` | Enable cross-storage backups (e.g., MinIO to AWS S3). | `false` | +| `backup` | `maxSegmentGroupSize` | Maximum segment group size for backups. | `2G` | +| | `parallelism.backupCollection` | Collection-level parallelism for backup. | `4` | +| | `parallelism.copydata` | Thread pool size for copying data. | `128` | +| | `parallelism.restoreCollection` | Collection-level parallelism for restore. | `2` | +| | `keepTempFiles` | Whether to keep temporary files during restore (for debugging). | `false` | +| | `gcPause.enable` | Pause Milvus garbage collection during backup. | `true` | +| | `gcPause.seconds` | Duration to pause garbage collection (in seconds). | `7200` | +| | `gcPause.address` | Address for Milvus garbage collection API. | `http://localhost:9091` | + +For more details, refer to the [backup.yaml](configs/backup.yaml) configuration file. ### Advanced feature @@ -93,6 +138,27 @@ http://localhost:8080/api/v1/docs/index.html 2. [RBAC Backup&Restore](docs/user_guide/rbac.md): Enable backup and restore RBAC meta with extra parameter. +## Examples + +### Syncing Minio Backups to an AWS S3 Bucket + +> **NOTE:** The following configuration is an example only. Replace the placeholders with your actual AWS and Minio settings. + +To back up Milvus data to an AWS S3 bucket, you need to configure the `backup.yaml` file with the following settings: + +```yaml +# Backup storage configs: Configure the storage where you want to save the backup data +backupStorageType: "aws" # Specifies the storage type as AWS S3 +backupAddress: s3.{your-aws-region}.amazonaws.com # Address of AWS S3 (replace {your-aws-region} with your bucket's region) +backupPort: 443 # Default port for AWS S3 +backupAccessKeyID: # Access key ID for your AWS S3 +backupSecretAccessKey: # Secret access key for AWS S3 +backupBucketName: "your-bucket-name" # Bucket name where the backups will be stored +backupRootPath: "backups" # Root path inside the bucket to store backups +backupUseSSL: true # Use SSL for secure connections (Required) +crossStorage: "true" # Required for minio to S3 backups +``` + ## Development From 9b590de14d5e3297a5771e3087cd297b97662f22 Mon Sep 17 00:00:00 2001 From: zhuwenxing Date: Fri, 29 Nov 2024 10:32:27 +0800 Subject: [PATCH 4/4] ci: add arm image publish (#458) Signed-off-by: zhuwenxing --- .github/workflows/release.yaml | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5f9b2f6..fbc5fcc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,6 +35,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} GITHUB_HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }} + dockerhub: runs-on: ubuntu-latest needs: goreleaser @@ -47,13 +48,35 @@ jobs: with: go-version: 1.18 cache: true + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Docker image - shell: bash + + - name: Get build info + id: build_info run: | - bash build_image.sh - docker push milvusdb/milvus-backup:$(git describe --tags --always) - docker push milvusdb/milvus-backup:latest + echo "VERSION=$(git describe --tags --always)" >> $GITHUB_OUTPUT + echo "COMMIT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + milvusdb/milvus-backup:${{ steps.build_info.outputs.VERSION }} + milvusdb/milvus-backup:latest + build-args: | + VERSION=${{ steps.build_info.outputs.VERSION }} + COMMIT=${{ steps.build_info.outputs.COMMIT }} + DATE=${{ steps.build_info.outputs.DATE }}