Merge pull request thanos-io#155 from fpetkovski/bucket-provider

Add Provider() method to objstore.Client

CHANGELOG.md

@@ -25,7 +25,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations.
 
 ### Added
-- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client.
 - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support.
 - [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class.
 - [#32](https://github.com/thanos-io/objstore/pull/32) Swift: Support authentication using application credentials.
@@ -56,6 +55,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket
 - [#147](https://github.com/thanos-io/objstore/pull/147) feat: Add MaxRetries config to cos, gcs and obs.
 - [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper.
+- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client.
+- [#155](https://github.com/thanos-io/objstore/pull/155) Add a `Provider` method on `objstore.Client`.
+
 
 ### Changed
 - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`.

README.md

@@ -48,13 +48,15 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m
 
 The core this module is the [`Bucket` interface](objstore.go):
 
-```go mdox-exec="sed -n '39,55p' objstore.go"
+```go mdox-exec="sed -n '55,73p' objstore.go"
 // Bucket provides read and write access to an object storage bucket.
 // NOTE: We assume strong consistency for write-read flow.
 type Bucket interface {
 	io.Closer
 	BucketReader
 
+	Provider() ObjProvider
+
 	// Upload the contents of the reader as an object into the bucket.
 	// Upload should be idempotent.
 	Upload(ctx context.Context, name string, r io.Reader) error
@@ -70,7 +72,7 @@ type Bucket interface {
 
 All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go):
 
-```go mdox-exec="sed -n '71,106p' objstore.go"
+```go mdox-exec="sed -n '89,124p' objstore.go"
 // BucketReader provides read access to an object storage bucket.
 type BucketReader interface {
 	// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full

client/factory.go

@@ -27,25 +27,10 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-type ObjProvider string
-
-const (
-	FILESYSTEM ObjProvider = "FILESYSTEM"
-	GCS        ObjProvider = "GCS"
-	S3         ObjProvider = "S3"
-	AZURE      ObjProvider = "AZURE"
-	SWIFT      ObjProvider = "SWIFT"
-	COS        ObjProvider = "COS"
-	ALIYUNOSS  ObjProvider = "ALIYUNOSS"
-	BOS        ObjProvider = "BOS"
-	OCI        ObjProvider = "OCI"
-	OBS        ObjProvider = "OBS"
-)
-
 type BucketConfig struct {
-	Type   ObjProvider `yaml:"type"`
-	Config interface{} `yaml:"config"`
-	Prefix string      `yaml:"prefix" default:""`
+	Type   objstore.ObjProvider `yaml:"type"`
+	Config interface{}          `yaml:"config"`
+	Prefix string               `yaml:"prefix" default:""`
 }
 
 // NewBucket initializes and returns new object storage clients.
@@ -64,25 +49,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrap
 
 	var bucket objstore.Bucket
 	switch strings.ToUpper(string(bucketConf.Type)) {
-	case string(GCS):
+	case string(objstore.GCS):
 		bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper)
-	case string(S3):
+	case string(objstore.S3):
 		bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper)
-	case string(AZURE):
+	case string(objstore.AZURE):
 		bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper)
-	case string(SWIFT):
+	case string(objstore.SWIFT):
 		bucket, err = swift.NewContainer(logger, config, wrapRoundtripper)
-	case string(COS):
+	case string(objstore.COS):
 		bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper)
-	case string(ALIYUNOSS):
+	case string(objstore.ALIYUNOSS):
 		bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper)
-	case string(FILESYSTEM):
+	case string(objstore.FILESYSTEM):
 		bucket, err = filesystem.NewBucketFromConfig(config)
-	case string(BOS):
+	case string(objstore.BOS):
 		bucket, err = bos.NewBucket(logger, config, component)
-	case string(OCI):
+	case string(objstore.OCI):
 		bucket, err = oci.NewBucket(logger, config, wrapRoundtripper)
-	case string(OBS):
+	case string(objstore.OBS):
 		bucket, err = obs.NewBucket(logger, config)
 	default:
 		return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type)

inmem.go

@@ -34,6 +34,8 @@ func NewInMemBucket() *InMemBucket {
 	}
 }
 
+func (b *InMemBucket) Provider() ObjProvider { return MEMORY }
+
 // Objects returns a copy of the internally stored objects.
 // NOTE: For assert purposes.
 func (b *InMemBucket) Objects() map[string][]byte {

objstore.go

@@ -26,6 +26,22 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+type ObjProvider string
+
+const (
+	MEMORY     ObjProvider = "MEMORY"
+	FILESYSTEM ObjProvider = "FILESYSTEM"
+	GCS        ObjProvider = "GCS"
+	S3         ObjProvider = "S3"
+	AZURE      ObjProvider = "AZURE"
+	SWIFT      ObjProvider = "SWIFT"
+	COS        ObjProvider = "COS"
+	ALIYUNOSS  ObjProvider = "ALIYUNOSS"
+	BOS        ObjProvider = "BOS"
+	OCI        ObjProvider = "OCI"
+	OBS        ObjProvider = "OBS"
+)
+
 const (
 	OpIter       = "iter"
 	OpGet        = "get"
@@ -42,6 +58,8 @@ type Bucket interface {
 	io.Closer
 	BucketReader
 
+	Provider() ObjProvider
+
 	// Upload the contents of the reader as an object into the bucket.
 	// Upload should be idempotent.
 	Upload(ctx context.Context, name string, r io.Reader) error
@@ -583,6 +601,10 @@ type metricBucket struct {
 	metrics *Metrics
 }
 
+func (b *metricBucket) Provider() ObjProvider {
+	return b.bkt.Provider()
+}
+
 func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket {
 	return &metricBucket{
 		bkt: b.bkt,

objtesting/foreach.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	"github.com/thanos-io/objstore"
-	"github.com/thanos-io/objstore/client"
 	"github.com/thanos-io/objstore/providers/azure"
 	"github.com/thanos-io/objstore/providers/bos"
 	"github.com/thanos-io/objstore/providers/cos"
@@ -26,7 +25,7 @@ import (
 
 // IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g:
 // THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI.
-func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool {
+func IsObjStoreSkipped(t *testing.T, provider objstore.ObjProvider) bool {
 	if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok {
 		obstores := strings.Split(e, ",")
 		for _, objstore := range obstores {
@@ -69,7 +68,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	})
 
 	// Optional GCS.
-	if !IsObjStoreSkipped(t, client.GCS) {
+	if !IsObjStoreSkipped(t, objstore.GCS) {
 		t.Run("gcs", func(t *testing.T) {
 			bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
 			testutil.Ok(t, err)
@@ -84,7 +83,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional S3.
-	if !IsObjStoreSkipped(t, client.S3) {
+	if !IsObjStoreSkipped(t, objstore.S3) {
 		t.Run("aws s3", func(t *testing.T) {
 			// TODO(bwplotka): Allow taking location from envvar.
 			bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
@@ -103,7 +102,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional Azure.
-	if !IsObjStoreSkipped(t, client.AZURE) {
+	if !IsObjStoreSkipped(t, objstore.AZURE) {
 		t.Run("azure", func(t *testing.T) {
 			bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
 			testutil.Ok(t, err)
@@ -117,7 +116,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional SWIFT.
-	if !IsObjStoreSkipped(t, client.SWIFT) {
+	if !IsObjStoreSkipped(t, objstore.SWIFT) {
 		t.Run("swift", func(t *testing.T) {
 			container, closeFn, err := swift.NewTestContainer(t)
 			testutil.Ok(t, err)
@@ -131,7 +130,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional COS.
-	if !IsObjStoreSkipped(t, client.COS) {
+	if !IsObjStoreSkipped(t, objstore.COS) {
 		t.Run("Tencent cos", func(t *testing.T) {
 			bkt, closeFn, err := cos.NewTestBucket(t)
 			testutil.Ok(t, err)
@@ -145,7 +144,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional OSS.
-	if !IsObjStoreSkipped(t, client.ALIYUNOSS) {
+	if !IsObjStoreSkipped(t, objstore.ALIYUNOSS) {
 		t.Run("AliYun oss", func(t *testing.T) {
 			bkt, closeFn, err := oss.NewTestBucket(t)
 			testutil.Ok(t, err)
@@ -159,7 +158,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional BOS.
-	if !IsObjStoreSkipped(t, client.BOS) {
+	if !IsObjStoreSkipped(t, objstore.BOS) {
 		t.Run("Baidu BOS", func(t *testing.T) {
 			bkt, closeFn, err := bos.NewTestBucket(t)
 			testutil.Ok(t, err)
@@ -173,7 +172,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional OCI.
-	if !IsObjStoreSkipped(t, client.OCI) {
+	if !IsObjStoreSkipped(t, objstore.OCI) {
 		t.Run("oci", func(t *testing.T) {
 			bkt, closeFn, err := oci.NewTestBucket(t)
 			testutil.Ok(t, err)
@@ -186,7 +185,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket))
 	}
 
 	// Optional OBS.
-	if !IsObjStoreSkipped(t, client.OBS) {
+	if !IsObjStoreSkipped(t, objstore.OBS) {
 		t.Run("obs", func(t *testing.T) {
 			bkt, closeFn, err := obs.NewTestBucket(t, "cn-south-1")
 			testutil.Ok(t, err)

prefixed_bucket.go

@@ -39,6 +39,8 @@ func withPrefix(prefix, name string) string {
 	return prefix + DirDelim + name
 }
 
+func (p *PrefixedBucket) Provider() ObjProvider { return p.bkt.Provider() }
+
 func (p *PrefixedBucket) Close() error {
 	return p.bkt.Close()
 }
@@ -93,7 +95,7 @@ func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool {
 }
 
 // Attributes returns information about the specified object.
-func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) {
+func (p *PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) {
 	return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name))
 }
 

providers/azure/azure.go

@@ -193,6 +193,8 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR
 	return bkt, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.AZURE }
+
 func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType {
 	return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt}
 }

providers/bos/bos.go

@@ -100,6 +100,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B
 	return bkt, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.BOS }
+
 // Name returns the bucket name for the provider.
 func (b *Bucket) Name() string {
 	return b.name

providers/cos/cos.go

@@ -159,6 +159,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra
 	return bkt, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.COS }
+
 // Name returns the bucket name for COS.
 func (b *Bucket) Name() string {
 	return b.name

providers/filesystem/filesystem.go

@@ -50,6 +50,8 @@ func NewBucket(rootDir string) (*Bucket, error) {
 	return &Bucket{rootDir: absDir}, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.FILESYSTEM }
+
 func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType {
 	return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt}
 }

providers/gcs/gcs.go

@@ -186,6 +186,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.
 	return bkt, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.GCS }
+
 // Name returns the bucket name for gcs.
 func (b *Bucket) Name() string {
 	return b.name

providers/obs/obs.go

@@ -122,6 +122,8 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) {
 	return bkt, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OBS }
+
 // Name returns the bucket name for the provider.
 func (b *Bucket) Name() string {
 	return b.name

providers/oci/oci.go

@@ -96,6 +96,8 @@ type Bucket struct {
 	requestMetadata common.RequestMetadata
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OCI }
+
 // Name returns the bucket name for the provider.
 func (b *Bucket) Name() string {
 	return b.name

providers/oss/oss.go

@@ -68,6 +68,8 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
 	return NewTestBucketFromConfig(t, c, false)
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.ALIYUNOSS }
+
 // Upload the contents of the reader as an object into the bucket.
 func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error {
 	// TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this.

providers/s3/s3.go

@@ -345,6 +345,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra
 	return bkt, nil
 }
 
+func (b *Bucket) Provider() objstore.ObjProvider { return objstore.S3 }
+
 // Name returns the bucket name for s3.
 func (b *Bucket) Name() string {
 	return b.name

providers/swift/swift.go

@@ -218,6 +218,8 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool,
 	}, nil
 }
 
+func (c *Container) Provider() objstore.ObjProvider { return objstore.SWIFT }
+
 // Name returns the container name for swift.
 func (c *Container) Name() string {
 	return c.name