gcpnative support and azure issue fixes

Signed-off-by: gifi.s <gifi.s@ibm.com>

cmd/backup_yaml.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"strings"
+
 	"github.com/spf13/cobra"
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"gopkg.in/yaml.v3"
@@ -39,20 +40,24 @@ type YAMLConFig struct {
 		TlsMode              int    `yaml:"tlsMode"`
 		User                 string `yaml:"user"`
 		Password             string `yaml:"password"`
+		tlsCertPath          string `yaml:"tlsCertPath"`
+		serverName           string `yaml:"serverName"`
 	} `yaml:"milvus"`
 	Minio struct {
-		Address          string `yaml:"address"`
-		Port             int    `yaml:"port"`
-		AccessKeyID      string `yaml:"accessKeyID"`
-		secretAccessKey  string `yaml:"secretAccessKey"`
-		UseSSL           bool   `yaml:"useSSL"`
-		UseIAM           bool   `yaml:"useIAM"`
-		CloudProvider    string `yaml:"cloudProvider"`
-		IamEndpoint      string `yaml:"iamEndpoint"`
-		BucketName       string `yaml:"bucketName"`
-		RootPath         string `yaml:"rootPath"`
-		BackupBucketName string `yaml:"backupBucketName"`
-		BackupRootPath   string `yaml:"backupRootPath"`
+		Address                 string `yaml:"address"`
+		Port                    int    `yaml:"port"`
+		AccessKeyID             string `yaml:"accessKeyID"`
+		secretAccessKey         string `yaml:"secretAccessKey"`
+		GcpCredentialJSON       string `yaml:"gcpCredentialJSON"`
+		UseSSL                  bool   `yaml:"useSSL"`
+		UseIAM                  bool   `yaml:"useIAM"`
+		CloudProvider           string `yaml:"cloudProvider"`
+		IamEndpoint             string `yaml:"iamEndpoint"`
+		BucketName              string `yaml:"bucketName"`
+		RootPath                string `yaml:"rootPath"`
+		BackupGcpCredentialJSON string `yaml:"backupGcpCredentialJSON"`
+		BackupBucketName        string `yaml:"backupBucketName"`
+		BackupRootPath          string `yaml:"backupRootPath"`
 	} `yaml:"minio"`
 	Backup struct {
 		MaxSegmentGroupSize string `yaml:"maxSegmentGroupSize"`
@@ -77,17 +82,21 @@ func printParams(base *paramtable.BackupParams) {
 	yml.Milvus.TlsMode = base.ParseIntWithDefault("milvus.tlsMode", 0)
 	yml.Milvus.User = base.BaseTable.LoadWithDefault("milvus.user", "")
 	yml.Milvus.Password = base.BaseTable.LoadWithDefault("milvus.password", "")
+	yml.Milvus.tlsCertPath = base.BaseTable.LoadWithDefault("milvus.tlsCertPath", "")
+	yml.Milvus.serverName = base.BaseTable.LoadWithDefault("milvus.serverName", "localhost")
 
 	yml.Minio.Address = base.LoadWithDefault("minio.address", "localhost")
 	yml.Minio.Port = base.ParseIntWithDefault("minio.port", 9000)
 	yml.Minio.AccessKeyID = base.BaseTable.LoadWithDefault("minio.accessKeyID", "")
 	yml.Minio.secretAccessKey = base.BaseTable.LoadWithDefault("minio.secretAccessKey", "")
+	yml.Minio.GcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.gcpCredentialJSON", "")
 	yml.Minio.UseSSL = base.ParseBool("minio.useSSL", false)
 	yml.Minio.UseIAM = base.ParseBool("minio.useIAM", false)
 	yml.Minio.CloudProvider = base.BaseTable.LoadWithDefault("minio.cloudProvider", "aws")
 	yml.Minio.IamEndpoint = base.BaseTable.LoadWithDefault("minio.iamEndpoint", "")
 	yml.Minio.BucketName = base.BaseTable.LoadWithDefault("minio.bucketName", "")
 	yml.Minio.RootPath = base.LoadWithDefault("minio.rootPath", "")
+	yml.Minio.BackupGcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.backupGcpCredentialJSON", "")
 	yml.Minio.BackupBucketName = base.LoadWithDefault("minio.backupBucketName", "")
 	yml.Minio.BackupRootPath = base.LoadWithDefault("minio.backupRootPath", "")
 

configs/backup.yaml

@@ -1,6 +1,6 @@
 # Configures the system log output.
 log:
-  level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
+  level: debug # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
   console: true # whether print log to console
   file:
     rootPath: "logs/backup.log"
@@ -12,33 +12,38 @@ http:
 milvus:
   address: localhost
   port: 19530
-  authorizationEnabled: false
+  authorizationEnabled: true
   # tls mode values [0, 1, 2]
   # 0 is close, 1 is one-way authentication, 2 is two-way authentication.
-  tlsMode: 0
+  tlsMode: 1
   user: "root"
   password: "Milvus"
+  tlsCertPath: "" # Path to certificate for authentication
+  serverName: ""
 
 # Related configuration of minio, which is responsible for data persistence for Milvus.
 minio:
   # Milvus storage configs, make them the same with milvus config
-  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
+  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative
+  # You can use "gcpnative" for Google Cloud Platform provider. Uses service account credentials for authentication.
   address: localhost # Address of MinIO/S3
   port: 9000   # Port of MinIO/S3
   accessKeyID: minioadmin  # accessKeyID of MinIO/S3
   secretAccessKey: minioadmin # MinIO/S3 encryption string
+  gcpCredentialJSON: "" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider.
   useSSL: false # Access to MinIO/S3 with SSL
   useIAM: false
   iamEndpoint: ""
   bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
   rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance
 
   # Backup storage configs, the storage you want to put the backup data
-  backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
+  backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative
   backupAddress: localhost # Address of MinIO/S3
   backupPort: 9000   # Port of MinIO/S3
   backupAccessKeyID: minioadmin  # accessKeyID of MinIO/S3
   backupSecretAccessKey: minioadmin # MinIO/S3 encryption string
+  backupGcpCredentialJSON: "" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider.
   backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath
   backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
 

core/backup_impl_create_backup.go

@@ -752,10 +752,10 @@ func (b *BackupContext) executeCreateBackup(ctx context.Context, request *backup
 
 func (b *BackupContext) writeBackupInfoMeta(ctx context.Context, id string) error {
 	backupInfo := b.meta.GetFullMeta(id)
-	log.Debug("Final backupInfo", zap.String("backupInfo", backupInfo.String()))
+	log.Info("Final backupInfo", zap.String("backupInfo", backupInfo.String()))
 	output, _ := serialize(backupInfo)
 	log.Debug("backup meta", zap.String("value", string(output.BackupMetaBytes)))
-	log.Debug("collection meta", zap.String("value", string(output.CollectionMetaBytes)))
+	log.Info("collection meta", zap.String("value", string(output.CollectionMetaBytes)))
 	log.Debug("partition meta", zap.String("value", string(output.PartitionMetaBytes)))
 	log.Debug("segment meta", zap.String("value", string(output.SegmentMetaBytes)))
 

core/backup_impl_restore_backup.go

@@ -689,16 +689,6 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
 			// group.files[0] += "453580537997766901/0/453580537997566904"
 			//group.files[0] = "gs://gcs-nv1/nativemill/BackupMilvus864/flavorBacks15/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/"
 			job := func(ctx context.Context) error {
-				// GIFI Start
-				// var slice []string
-				// slice = append(slice, "nativemill/BackupMilvus864/flavorBack5/")
-				// slice = append(slice, "nativemill/BackupMilvus864/flavorBack5/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/453580537997766901/1/")
-				// slice = append(slice, "nativemill/BackupMilvus864/flavorBack5/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/453580537997766901/100/")
-				// slice = append(slice, "nativemill/BackupMilvus864/flavorBack5/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/453580537997766901/101/")
-				// slice = append(slice, "nativemill/BackupMilvus864/flavorBack5/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/453580537997766901/102/")
-
-				// err := copyAndBulkInsert(targetDBName, targetCollectionName, partitionBackup.GetPartitionName(), slice, false, task.GetSkipDiskQuotaCheck())
-				// GIFI End
 				err := copyAndBulkInsert(targetDBName, targetCollectionName, partitionBackup.GetPartitionName(), group.files, false, task.GetSkipDiskQuotaCheck())
 				if err != nil {
 					return err
@@ -893,8 +883,8 @@ func (b *BackupContext) getBackupPartitionPathsWithGroupID(ctx context.Context,
 		zap.Int64("partitionID", partition.GetPartitionId()),
 		zap.Int64("groupId", groupId))
 
-	// insertPath := fmt.Sprintf("%s/%s/%s/%v/%v/%d/", backupPath, BINGLOG_DIR, INSERT_LOG_DIR, partition.GetCollectionId(), partition.GetPartitionId(), groupId)
-	insertPath := "nativemill/BackupMilvus864/flavorBack5/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/453580537997766901/333/" //GIFI
+	insertPath := fmt.Sprintf("%s/%s/%s/%v/%v/%d/", backupPath, BINGLOG_DIR, INSERT_LOG_DIR, partition.GetCollectionId(), partition.GetPartitionId(), groupId)
+	// insertPath := "nativemill/BackupMilvus864/flavorBack5/binlogs/insert_log/453580537997566884/453580537997566885/453580537997766901/453580537997766901/333/" //GIFI
 	deltaPath := fmt.Sprintf("%s/%s/%s/%v/%v/%d/", backupPath, BINGLOG_DIR, DELTA_LOG_DIR, partition.GetCollectionId(), partition.GetPartitionId(), groupId)
 	var totalSize int64
 	for _, seg := range partition.GetSegmentBackups() {

core/storage/azure_chunk_manager.go

@@ -313,8 +313,9 @@ func (mcm *AzureChunkManager) RemoveWithPrefix(ctx context.Context, bucketName s
 
 // ListWithPrefix returns objects with provided prefix.
 func (mcm *AzureChunkManager) ListWithPrefix(ctx context.Context, bucketName string, prefix string, recursive bool) ([]string, []int64, error) {
-	objects, err := mcm.listObjects(ctx, bucketName, prefix, true)
-
+	// objects, err := mcm.listObjects(ctx, bucketName, prefix, true) //GIFI
+	objects, err := mcm.listObjects(ctx, bucketName, prefix, false)
+	log.Info("GIFI ListWithPrefix------", zap.Any("prefix", prefix))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -332,9 +333,9 @@ func (mcm *AzureChunkManager) ListWithPrefix(ctx context.Context, bucketName str
 		objectsKeysDict := make(map[string]bool, 0)
 		for object, size := range objects {
 			keyWithoutPrefix := strings.Replace(object, prefix, "", 1)
-			// log.Info("------", zap.Any("object", object))
-			// log.Info("------", zap.Any("prefix", prefix))
-			// log.Info("------", zap.Any("keyWithoutPrefix", keyWithoutPrefix))
+			log.Info("GIFI------", zap.Any("object", object))
+			log.Info("GIFI------", zap.Any("prefix", prefix))
+			log.Info("GIFI------", zap.Any("keyWithoutPrefix", keyWithoutPrefix))
 			if strings.Contains(keyWithoutPrefix, "/") {
 				var key string
 				if strings.HasPrefix(keyWithoutPrefix, "/") {

core/storage/gcp_native_chunk_manager.go

@@ -56,10 +56,11 @@ func newGCPNativeChunkManagerWithConfig(ctx context.Context, config *StorageConf
 	var opts []option.ClientOption
 	var projectId string
 	if config.Address != "" {
-		complete_address := "http://"
-		if config.UseSSL {
-			complete_address = "https://"
-		}
+		// complete_address := "http://" //GIFI OLD
+		complete_address := "https://"
+		// if config.UseSSL {
+		// 	complete_address = "https://"
+		// }
 		complete_address = complete_address + config.Address + "/storage/v1/"
 		opts = append(opts, option.WithEndpoint(complete_address))
 	}

go.mod

@@ -10,42 +10,66 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cockroachdb/errors v1.9.1
 	github.com/gin-gonic/gin v1.8.1
-	github.com/golang/protobuf v1.5.2
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4
 	github.com/google/btree v1.0.1
-	github.com/google/uuid v1.3.0
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/uuid v1.6.0
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.5 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
 	//github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3
-	github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16
+	github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265
 	github.com/minio/minio-go/v7 v7.0.61
 	github.com/pkg/errors v0.9.1
 	github.com/samber/lo v1.39.0
 	github.com/sony/sonyflake v1.1.0
 	github.com/spf13/cast v1.3.1
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.8.1
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/files v1.0.0
 	github.com/swaggo/gin-swagger v1.5.3
 	github.com/swaggo/swag v1.16.1
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.865
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.uber.org/atomic v1.10.0
 	go.uber.org/zap v1.17.0
-	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
-	golang.org/x/sync v0.8.0
-	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
-	google.golang.org/grpc v1.48.0
+	// golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
+	golang.org/x/oauth2 v0.21.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/time v0.5.0
+	google.golang.org/api v0.187.0
+	google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
+	google.golang.org/grpc v1.64.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20241030091852-d6eb85c1a8ff
+require github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a
 
-require github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd
+require (
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect
+)
 
 require (
-	cloud.google.com/go v0.81.0 // indirect
+	//cloud.google.com/go v0.81.0 // indirect
+	cloud.google.com/go v0.115.0 // indirect
+	cloud.google.com/go/auth v0.6.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
+	cloud.google.com/go/iam v1.1.8 // indirect
+	cloud.google.com/go/storage v1.43.0
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
@@ -55,6 +79,7 @@ require (
 	github.com/cockroachdb/redact v1.1.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/getsentry/sentry-go v0.12.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -102,15 +127,15 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.11.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
+
+// replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20240814024750-1d41342d2c27