Support tencent cloud object storage

configs/backup.yaml

@@ -22,7 +22,7 @@ milvus:
 # Related configuration of minio, which is responsible for data persistence for Milvus.
 minio:
   # cloudProvider: "minio" # deprecated use storageType instead
-  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure
+  storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
 
   address: localhost # Address of MinIO/S3
   port: 9000   # Port of MinIO/S3

core/milvus_storage_test.go

@@ -7,8 +7,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/zilliztech/milvus-backup/core/storage/tencent"
 
 	"github.com/zilliztech/milvus-backup/core/paramtable"
 	"github.com/zilliztech/milvus-backup/core/storage"
@@ -113,3 +116,18 @@ func TestReadMilvusData(t *testing.T) {
 	}
 
 }
+
+func TestTencentOSS(t *testing.T) {
+	var creds *credentials.Credentials
+	bucketLookupType := minio.BucketLookupDNS
+	minioOpts := &minio.Options{
+		BucketLookup: bucketLookupType,
+		Creds:        creds,
+		Secure:       true,
+	}
+	client, err := tencent.NewMinioClient("cos.ap-nanjing.myqcloud.com", minioOpts)
+	assert.NoError(t, err)
+	exist, err := client.BucketExists(context.Background(), "")
+	assert.NoError(t, err)
+	assert.True(t, exist)
+}

core/paramtable/params.go

@@ -166,25 +166,29 @@ func (p *MilvusConfig) initTLSMode() {
 // /////////////////////////////////////////////////////////////////////////////
 // --- minio ---
 const (
-	Local               = "local"
-	Minio               = "minio"
-	S3                  = "s3"
-	CloudProviderAWS    = "aws"
-	CloudProviderGCP    = "gcp"
-	CloudProviderAli    = "ali"
-	CloudProviderAliyun = "aliyun"
-	CloudProviderAzure  = "azure"
+	Local                     = "local"
+	Minio                     = "minio"
+	S3                        = "s3"
+	CloudProviderAWS          = "aws"
+	CloudProviderGCP          = "gcp"
+	CloudProviderAli          = "ali"
+	CloudProviderAliyun       = "aliyun"
+	CloudProviderAzure        = "azure"
+	CloudProviderTencent      = "tencent"
+	CloudProviderTencentShort = "tc"
 )
 
 var supportedStorageType = map[string]bool{
-	Local:               true,
-	Minio:               true,
-	S3:                  true,
-	CloudProviderAWS:    true,
-	CloudProviderGCP:    true,
-	CloudProviderAli:    true,
-	CloudProviderAliyun: true,
-	CloudProviderAzure:  true,
+	Local:                     true,
+	Minio:                     true,
+	S3:                        true,
+	CloudProviderAWS:          true,
+	CloudProviderGCP:          true,
+	CloudProviderAli:          true,
+	CloudProviderAliyun:       true,
+	CloudProviderAzure:        true,
+	CloudProviderTencent:      true,
+	CloudProviderTencentShort: true,
 }
 
 type MinioConfig struct {

core/storage/minio_chunk_manager.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/zilliztech/milvus-backup/core/storage/tencent"
 	"golang.org/x/sync/errgroup"
 	"io"
 	"strings"
@@ -66,6 +67,13 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
 
 	switch c.storageType {
 	case paramtable.CloudProviderAliyun:
+		// auto doesn't work for aliyun, so we set to dns deliberately
+		bucketLookupType = minio.BucketLookupDNS
+		if c.useIAM {
+			newMinioFn = aliyun.NewMinioClient
+		} else {
+			creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "")
+		}
 	case paramtable.CloudProviderAli:
 		// auto doesn't work for aliyun, so we set to dns deliberately
 		bucketLookupType = minio.BucketLookupDNS
@@ -79,6 +87,18 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
 		if !c.useIAM {
 			creds = credentials.NewStaticV2(c.accessKeyID, c.secretAccessKeyID, "")
 		}
+	case paramtable.CloudProviderTencentShort:
+		bucketLookupType = minio.BucketLookupDNS
+		newMinioFn = tencent.NewMinioClient
+		if !c.useIAM {
+			creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "")
+		}
+	case paramtable.CloudProviderTencent:
+		bucketLookupType = minio.BucketLookupDNS
+		newMinioFn = tencent.NewMinioClient
+		if !c.useIAM {
+			creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "")
+		}
 	default: // aws, minio
 		if c.useIAM {
 			creds = credentials.NewIAM("")

core/storage/tencent/tencent.go

@@ -0,0 +1,85 @@
+package tencent
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/errors"
+	"github.com/minio/minio-go/v7"
+	minioCred "github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
+)
+
+// NewMinioClient returns a minio.Client which is compatible for tencent OSS
+func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error) {
+	if opts == nil {
+		opts = &minio.Options{}
+	}
+	if opts.Creds == nil {
+		credProvider, err := NewCredentialProvider()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create credential provider")
+		}
+		opts.Creds = minioCred.New(credProvider)
+	}
+	if address == "" {
+		address = fmt.Sprintf("cos.%s.myqcloud.com", opts.Region)
+		opts.Secure = true
+	}
+	return minio.New(address, opts)
+}
+
+// Credential is defined to mock tencent credential.Credentials
+//
+//go:generate mockery --name=Credential --with-expecter
+type Credential interface {
+	common.CredentialIface
+}
+
+// CredentialProvider implements "github.com/minio/minio-go/v7/pkg/credentials".Provider
+// also implements transport
+type CredentialProvider struct {
+	// tencentCreds doesn't provide a way to get the expired time, so we use the cache to check if it's expired
+	// when tencentCreds.GetSecretId is different from the cache, we know it's expired
+	akCache      string
+	tencentCreds Credential
+}
+
+func NewCredentialProvider() (minioCred.Provider, error) {
+	provider, err := common.DefaultTkeOIDCRoleArnProvider()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create tencent credential provider")
+	}
+
+	cred, err := provider.GetCredential()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get tencent credential")
+	}
+	return &CredentialProvider{tencentCreds: cred}, nil
+}
+
+// Retrieve returns nil if it successfully retrieved the value.
+// Error is returned if the value were not obtainable, or empty.
+// according to the caller minioCred.Credentials.Get(),
+// it already has a lock, so we don't need to worry about concurrency
+func (c *CredentialProvider) Retrieve() (minioCred.Value, error) {
+	ret := minioCred.Value{}
+	ak := c.tencentCreds.GetSecretId()
+	ret.AccessKeyID = ak
+	c.akCache = ak
+
+	sk := c.tencentCreds.GetSecretKey()
+	ret.SecretAccessKey = sk
+
+	securityToken := c.tencentCreds.GetToken()
+	ret.SessionToken = securityToken
+	return ret, nil
+}
+
+// IsExpired returns if the credentials are no longer valid, and need
+// to be retrieved.
+// according to the caller minioCred.Credentials.IsExpired(),
+// it already has a lock, so we don't need to worry about concurrency
+func (c CredentialProvider) IsExpired() bool {
+	ak := c.tencentCreds.GetSecretId()
+	return ak != c.akCache
+}

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
 	github.com/milvus-io/milvus-sdk-go/v2 v2.3.0
-	github.com/minio/minio-go/v7 v7.0.17
+	github.com/minio/minio-go/v7 v7.0.61
 	github.com/pkg/errors v0.9.1
 	github.com/samber/lo v1.39.0
 	github.com/sony/sonyflake v1.1.0
@@ -27,6 +27,7 @@ require (
 	github.com/swaggo/files v1.0.0
 	github.com/swaggo/gin-swagger v1.5.3
 	github.com/swaggo/swag v1.16.1
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.uber.org/atomic v1.10.0
@@ -50,7 +51,7 @@ require (
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/getsentry/sentry-go v0.12.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -68,8 +69,8 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.13.5 // indirect
-	github.com/klauspost/cpuid v1.3.1 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
@@ -78,9 +79,8 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.6 // indirect
-	github.com/minio/md5-simd v1.1.0 // indirect
-	github.com/minio/sha256-simd v0.1.1 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -89,11 +89,12 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	github.com/rs/xid v1.2.1 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/rs/xid v1.5.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
@@ -111,7 +112,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )