Skip to content

Commit

Permalink
Support tencent cloud object storage
Browse files Browse the repository at this point in the history
Signed-off-by: wayblink <anyang.wang@zilliz.com>
  • Loading branch information
wayblink committed Feb 22, 2024
1 parent c7e660a commit 93c5f7d
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 17 deletions.
2 changes: 1 addition & 1 deletion configs/backup.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ milvus:
# Related configuration of minio, which is responsible for data persistence for Milvus.
minio:
# cloudProvider: "minio" # deprecated use storageType instead
storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure
storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)

address: localhost # Address of MinIO/S3
port: 9000 # Port of MinIO/S3
Expand Down
36 changes: 20 additions & 16 deletions core/paramtable/params.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,25 +166,29 @@ func (p *MilvusConfig) initTLSMode() {
// /////////////////////////////////////////////////////////////////////////////
// --- minio ---
const (
Local = "local"
Minio = "minio"
S3 = "s3"
CloudProviderAWS = "aws"
CloudProviderGCP = "gcp"
CloudProviderAli = "ali"
CloudProviderAliyun = "aliyun"
CloudProviderAzure = "azure"
Local = "local"
Minio = "minio"
S3 = "s3"
CloudProviderAWS = "aws"
CloudProviderGCP = "gcp"
CloudProviderAli = "ali"
CloudProviderAliyun = "aliyun"
CloudProviderAzure = "azure"
CloudProviderTencent = "tencent"
CloudProviderTencentShort = "tc"
)

var supportedStorageType = map[string]bool{
Local: true,
Minio: true,
S3: true,
CloudProviderAWS: true,
CloudProviderGCP: true,
CloudProviderAli: true,
CloudProviderAliyun: true,
CloudProviderAzure: true,
Local: true,
Minio: true,
S3: true,
CloudProviderAWS: true,
CloudProviderGCP: true,
CloudProviderAli: true,
CloudProviderAliyun: true,
CloudProviderAzure: true,
CloudProviderTencent: true,
CloudProviderTencentShort: true,
}

type MinioConfig struct {
Expand Down
8 changes: 8 additions & 0 deletions core/storage/minio_chunk_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"github.com/zilliztech/milvus-backup/core/storage/tencent"
"golang.org/x/sync/errgroup"
"io"
"strings"
Expand Down Expand Up @@ -79,6 +80,13 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
if !c.useIAM {
creds = credentials.NewStaticV2(c.accessKeyID, c.secretAccessKeyID, "")
}
case paramtable.CloudProviderTencentShort:
case paramtable.CloudProviderTencent:
bucketLookupType = minio.BucketLookupDNS
newMinioFn = tencent.NewMinioClient
if !c.useIAM {
creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "")
}
default: // aws, minio
if c.useIAM {
creds = credentials.NewIAM("")
Expand Down
85 changes: 85 additions & 0 deletions core/storage/tencent/tencent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package tencent

import (
"fmt"

"github.com/cockroachdb/errors"
"github.com/minio/minio-go/v7"
minioCred "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
)

// NewMinioClient returns a minio.Client which is compatible for tencent OSS
func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error) {
if opts == nil {
opts = &minio.Options{}
}
if opts.Creds == nil {
credProvider, err := NewCredentialProvider()
if err != nil {
return nil, errors.Wrap(err, "failed to create credential provider")
}
opts.Creds = minioCred.New(credProvider)
}
if address == "" {
address = fmt.Sprintf("cos.%s.myqcloud.com", opts.Region)
opts.Secure = true
}
return minio.New(address, opts)
}

// Credential is defined to mock tencent credential.Credentials
//
//go:generate mockery --name=Credential --with-expecter
type Credential interface {
common.CredentialIface
}

// CredentialProvider implements "github.com/minio/minio-go/v7/pkg/credentials".Provider
// also implements transport
type CredentialProvider struct {
// tencentCreds doesn't provide a way to get the expired time, so we use the cache to check if it's expired
// when tencentCreds.GetSecretId is different from the cache, we know it's expired
akCache string
tencentCreds Credential
}

func NewCredentialProvider() (minioCred.Provider, error) {
provider, err := common.DefaultTkeOIDCRoleArnProvider()
if err != nil {
return nil, errors.Wrap(err, "failed to create tencent credential provider")
}

cred, err := provider.GetCredential()
if err != nil {
return nil, errors.Wrap(err, "failed to get tencent credential")
}
return &CredentialProvider{tencentCreds: cred}, nil
}

// Retrieve returns nil if it successfully retrieved the value.
// Error is returned if the value were not obtainable, or empty.
// according to the caller minioCred.Credentials.Get(),
// it already has a lock, so we don't need to worry about concurrency
func (c *CredentialProvider) Retrieve() (minioCred.Value, error) {
ret := minioCred.Value{}
ak := c.tencentCreds.GetSecretId()
ret.AccessKeyID = ak
c.akCache = ak

sk := c.tencentCreds.GetSecretKey()
ret.SecretAccessKey = sk

securityToken := c.tencentCreds.GetToken()
ret.SessionToken = securityToken
return ret, nil
}

// IsExpired returns if the credentials are no longer valid, and need
// to be retrieved.
// according to the caller minioCred.Credentials.IsExpired(),
// it already has a lock, so we don't need to worry about concurrency
func (c CredentialProvider) IsExpired() bool {
ak := c.tencentCreds.GetSecretId()
return ak != c.akCache
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ require (
github.com/swaggo/files v1.0.0
github.com/swaggo/gin-swagger v1.5.3
github.com/swaggo/swag v1.16.1
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841
github.com/uber/jaeger-client-go v2.25.0+incompatible
go.etcd.io/etcd/client/v3 v3.5.0
go.uber.org/atomic v1.10.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,8 @@ github.com/swaggo/gin-swagger v1.5.3/go.mod h1:3XJKSfHjDMB5dBo/0rrTXidPmgLeqsX89
github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
github.com/swaggo/swag v1.16.1 h1:fTNRhKstPKxcnoKsytm4sahr8FaYzUcT7i1/3nd/fBg=
github.com/swaggo/swag v1.16.1/go.mod h1:9/LMvHycG3NFHfR6LwvikHv5iFvmPADQ359cKikGxto=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841 h1:SJJR4tLnr0V17uEVS+arAmR1yl8n6dObBZs77SAmXZE=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
Expand Down

0 comments on commit 93c5f7d

Please sign in to comment.