This repository has been archived by the owner on Jul 19, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 73
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #437 from scottzhlin/feature/support-cos-objstore-…
…provider feat(objstore): support Tencent COS object storage
- Loading branch information
Showing
7 changed files
with
288 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package cos | ||
|
|
||
| import ( | ||
| "github.com/go-kit/log" | ||
| "github.com/prometheus/common/model" | ||
| "github.com/thanos-io/objstore" | ||
| "github.com/thanos-io/objstore/exthttp" | ||
| "github.com/thanos-io/objstore/providers/cos" | ||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| // NewBucketClient creates a bucket client for COS | ||
| func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { | ||
| bucketConfig := &cos.Config{ | ||
| Bucket: cfg.Bucket, | ||
| Region: cfg.Region, | ||
| AppId: cfg.AppID, | ||
| Endpoint: cfg.Endpoint, | ||
| SecretKey: cfg.SecretKey, | ||
| SecretId: cfg.SecretID, | ||
| HTTPConfig: exthttp.HTTPConfig{ | ||
| IdleConnTimeout: model.Duration(cfg.HTTP.IdleConnTimeout), | ||
| ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout), | ||
| InsecureSkipVerify: cfg.HTTP.InsecureSkipVerify, | ||
| TLSHandshakeTimeout: model.Duration(cfg.HTTP.TLSHandshakeTimeout), | ||
| ExpectContinueTimeout: model.Duration(cfg.HTTP.ExpectContinueTimeout), | ||
| MaxIdleConns: cfg.HTTP.MaxIdleConns, | ||
| MaxIdleConnsPerHost: cfg.HTTP.MaxIdleConnsPerHost, | ||
| MaxConnsPerHost: cfg.HTTP.MaxConnsPerHost, | ||
| Transport: cfg.HTTP.Transport, | ||
| }, | ||
| } | ||
|
|
||
| serializedConfig, err := yaml.Marshal(bucketConfig) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return cos.NewBucket(logger, serializedConfig, name) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package cos | ||
|
|
||
| import ( | ||
| "errors" | ||
| "flag" | ||
| "fmt" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
| ) | ||
|
|
||
| // Config encapsulates the necessary config values to instantiate an cos client. | ||
| type Config struct { | ||
| Bucket string `yaml:"bucket"` | ||
| Region string `yaml:"region"` | ||
| AppID string `yaml:"app_id"` | ||
| Endpoint string `yaml:"endpoint"` | ||
| SecretKey string `yaml:"secret_key"` | ||
| SecretID string `yaml:"secret_id"` | ||
| HTTP HTTPConfig `yaml:"http"` | ||
| } | ||
|
|
||
| // Validate validates cos client config and returns error on failure | ||
| func (c *Config) Validate() error { | ||
| if len(c.Endpoint) != 0 { | ||
| if _, err := url.Parse(c.Endpoint); err != nil { | ||
| return fmt.Errorf("cos config: failed to parse endpoint: %w", err) | ||
| } | ||
|
|
||
| if empty(c.SecretKey) || empty(c.SecretID) { | ||
| return errors.New("secret id and secret key cannot be empty") | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| if empty(c.Bucket) || empty(c.AppID) || empty(c.Region) || empty(c.SecretID) || empty(c.SecretKey) { | ||
| return errors.New("invalid cos configuration, bucket, app_id, region, secret_id and secret_key must be set") | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func empty(s string) bool { | ||
| return len(s) == 0 | ||
| } | ||
|
|
||
| // RegisterFlags registers the flags for COS storage | ||
| func (c *Config) RegisterFlags(f *flag.FlagSet) { | ||
| c.RegisterFlagsWithPrefix("", f) | ||
| } | ||
|
|
||
| // RegisterFlagsWithPrefix register the flags for COS storage with provided prefix | ||
| func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { | ||
| f.StringVar(&c.Bucket, prefix+"cos.bucket", "", "COS bucket name") | ||
| f.StringVar(&c.Region, prefix+"cos.region", "", "COS region name") | ||
| f.StringVar(&c.AppID, prefix+"cos.app-id", "", "COS app id") | ||
| f.StringVar(&c.Endpoint, prefix+"cos.endpoint", "", "COS storage endpoint") | ||
| f.StringVar(&c.SecretID, prefix+"cos.secret-id", "", "COS secret id") | ||
| f.StringVar(&c.SecretKey, prefix+"cos.secret-key", "", "COS secret key") | ||
| c.HTTP.RegisterFlagsWithPrefix(prefix, f) | ||
| } | ||
|
|
||
| // HTTPConfig stores the http.Transport configuration for the COS client. | ||
| type HTTPConfig struct { | ||
| IdleConnTimeout time.Duration `yaml:"idle_conn_timeout" category:"advanced"` | ||
| ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout" category:"advanced"` | ||
| InsecureSkipVerify bool `yaml:"insecure_skip_verify" category:"advanced"` | ||
| TLSHandshakeTimeout time.Duration `yaml:"tls_handshake_timeout" category:"advanced"` | ||
| ExpectContinueTimeout time.Duration `yaml:"expect_continue_timeout" category:"advanced"` | ||
| MaxIdleConns int `yaml:"max_idle_connections" category:"advanced"` | ||
| MaxIdleConnsPerHost int `yaml:"max_idle_connections_per_host" category:"advanced"` | ||
| MaxConnsPerHost int `yaml:"max_connections_per_host" category:"advanced"` | ||
|
|
||
| // Allow upstream callers to inject a round tripper | ||
| Transport http.RoundTripper `yaml:"-"` | ||
| } | ||
|
|
||
| // RegisterFlagsWithPrefix registers the flags for COS storage with the provided prefix | ||
| func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { | ||
| f.DurationVar(&cfg.IdleConnTimeout, prefix+"cos.http.idle-conn-timeout", 90*time.Second, "The time an idle connection will remain idle before closing.") | ||
| f.DurationVar(&cfg.ResponseHeaderTimeout, prefix+"cos.http.response-header-timeout", 2*time.Minute, "The amount of time the client will wait for a servers response headers.") | ||
| f.BoolVar(&cfg.InsecureSkipVerify, prefix+"cos.http.insecure-skip-verify", false, "If the client connects to COS via HTTPS and this option is enabled, the client will accept any certificate and hostname.") | ||
| f.DurationVar(&cfg.TLSHandshakeTimeout, prefix+"cos.tls-handshake-timeout", 10*time.Second, "Maximum time to wait for a TLS handshake. 0 means no limit.") | ||
| f.DurationVar(&cfg.ExpectContinueTimeout, prefix+"cos.expect-continue-timeout", 1*time.Second, "The time to wait for a server's first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately.") | ||
| f.IntVar(&cfg.MaxIdleConns, prefix+"cos.max-idle-connections", 100, "Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.") | ||
| f.IntVar(&cfg.MaxIdleConnsPerHost, prefix+"cos.max-idle-connections-per-host", 100, "Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.") | ||
| f.IntVar(&cfg.MaxConnsPerHost, prefix+"cos.max-connections-per-host", 0, "Maximum number of connections per host. 0 means no limit.") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package cos | ||
|
|
||
| import "testing" | ||
|
|
||
| func TestConfig_Validate(t *testing.T) { | ||
| type fields struct { | ||
| Bucket string | ||
| Region string | ||
| AppID string | ||
| Endpoint string | ||
| SecretKey string | ||
| SecretID string | ||
| HTTP HTTPConfig | ||
| } | ||
| tests := []struct { | ||
| name string | ||
| fields fields | ||
| wantErr bool | ||
| }{ | ||
| { | ||
| name: "ok endpoint", | ||
| fields: fields{ | ||
| Endpoint: "http://bucket-123.cos.ap-beijing.myqcloud.com", | ||
| SecretID: "sid", | ||
| SecretKey: "skey", | ||
| }, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "ok bucket-AppID-region", | ||
| fields: fields{ | ||
| Bucket: "bucket", | ||
| AppID: "123", | ||
| Region: "ap-beijing", | ||
| SecretID: "sid", | ||
| SecretKey: "skey", | ||
| }, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "missing skey", | ||
| fields: fields{ | ||
| Bucket: "bucket", | ||
| AppID: "123", | ||
| Region: "ap-beijing", | ||
| }, | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "missing bucket", | ||
| fields: fields{ | ||
| AppID: "123", | ||
| Region: "ap-beijing", | ||
| SecretID: "sid", | ||
| SecretKey: "skey", | ||
| }, | ||
| wantErr: true, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| c := &Config{ | ||
| Bucket: tt.fields.Bucket, | ||
| Region: tt.fields.Region, | ||
| AppID: tt.fields.AppID, | ||
| Endpoint: tt.fields.Endpoint, | ||
| SecretKey: tt.fields.SecretKey, | ||
| SecretID: tt.fields.SecretID, | ||
| HTTP: tt.fields.HTTP, | ||
| } | ||
| if err := c.Validate(); (err != nil) != tt.wantErr { | ||
| t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) | ||
| } | ||
| }) | ||
| } | ||
| } |