From 41ae41a80613a93bc39a8c87e3b43dcf1bb79474 Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Thu, 3 Feb 2022 10:04:42 +0100 Subject: [PATCH 1/4] Add a pure aws-sdk-go auth Signed-off-by: Sylvain Rabot --- CHANGELOG.md | 2 ++ docs/storage.md | 3 ++ go.mod | 9 +++++ go.sum | 19 +++++++++++ pkg/objstore/s3/s3.go | 12 ++++++- pkg/objstore/s3/s3_aws_sdk_auth.go | 54 ++++++++++++++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 pkg/objstore/s3/s3_aws_sdk_auth.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c54c6da84..c7d8319ac2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased +- [#4667](https://github.com/thanos-io/thanos/pull/4667) Add a pure aws-sdk auth for s3 storage. + ### Added - [#5110](https://github.com/thanos-io/thanos/pull/5110) Block: Do not upload DebugMeta files to obj store. diff --git a/docs/storage.md b/docs/storage.md index 87e9675057..7e10f5c98c 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -65,6 +65,7 @@ config: bucket: "" endpoint: "" region: "" + aws_sdk_auth: false access_key: "" insecure: false signature_version2: false @@ -99,6 +100,8 @@ config: At a minimum, you will need to provide a value for the `bucket`, `endpoint`, `access_key`, and `secret_key` keys. The rest of the keys are optional. +However if you set `aws_sdk_auth: true` Thanos will use the default authentication methods of the AWS SDK for go based on [known environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) (`AWS_PROFILE`, `AWS_WEB_IDENTITY_TOKEN_FILE` ... etc) and known AWS config files (~/.aws/config). If you turn this on, then the `bucket` and `endpoint` are the required config keys. + The AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/s3.html). Make sure you use a correct signature version. Currently AWS requires signature v4, so it needs `signature_version2: false`. If you don't specify it, you will get an `Access Denied` error. On the other hand, several S3 compatible APIs use `signature_version2: true`. diff --git a/go.mod b/go.mod index 797cc1d321..a573359e54 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a github.com/alicebob/miniredis/v2 v2.14.3 github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible + github.com/aws/aws-sdk-go-v2 v1.11.1 + github.com/aws/aws-sdk-go-v2/config v1.8.1 github.com/baidubce/bce-sdk-go v0.9.81 github.com/blang/semver/v4 v4.0.0 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b @@ -105,6 +107,13 @@ require ( github.com/armon/go-radix v1.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/aws/aws-sdk-go v1.42.8 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect + github.com/aws/smithy-go v1.9.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 // indirect diff --git a/go.sum b/go.sum index abca77e891..89e39d09e9 100644 --- a/go.sum +++ b/go.sum @@ -241,9 +241,28 @@ github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go v1.42.8 h1:Tj2RP4Fas1mYchwbmw0qWLJIEATAseyp5iTa1D+LWYQ= github.com/aws/aws-sdk-go v1.42.8/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2 v1.11.1 h1:GzvOVAdTbWxhEMRK4FfiblkGverOkAT0UodDxC1jHQM= +github.com/aws/aws-sdk-go-v2 v1.11.1/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/config v1.8.1 h1:AcAenV2NVwOViG+3ts73uT08L1olN4NBNNz7lUlHSUo= +github.com/aws/aws-sdk-go-v2/config v1.8.1/go.mod h1:AQtpYfVYjuuft4Dgh0jGSkPQJ9MvmK9vXfSub7oSXlI= +github.com/aws/aws-sdk-go-v2/credentials v1.4.1 h1:oDiUP50hKRwC6xAgESAj46lgL2prJRZQWnCBzn+TU/c= +github.com/aws/aws-sdk-go-v2/credentials v1.4.1/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 h1:OxTAgH8Y4BXHD6PGCJ8DHx2kaZPCQfSTqmDsdRZFezE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 h1:d95cddM3yTm4qffj3P6EnP+TzX1SSkWaQypXSgT/hpA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/baidubce/bce-sdk-go v0.9.81 h1:n8KfThLG9fvGv3A+RtTt/jKhg/FPPRpo+iNnS2r+iPI= github.com/baidubce/bce-sdk-go v0.9.81/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index ff10b52c29..a47d0113b8 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -74,6 +74,7 @@ type Config struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` Region string `yaml:"region"` + AWSSDKAuth bool `yaml:"aws_sdk_auth"` AccessKey string `yaml:"access_key"` Insecure bool `yaml:"insecure"` SignatureV2 bool `yaml:"signature_version2"` @@ -224,7 +225,16 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B if err := validate(config); err != nil { return nil, err } - if config.AccessKey != "" { + + if config.AWSSDKAuth && config.AccessKey != "" { + return nil, errors.New("aws_sdk_auth and access_key are mutually exclusive configurations") + } + + if config.AWSSDKAuth { + chain = []credentials.Provider{ + wrapCredentialsProvider(&AWSSDKAuth{Region: config.Region}), + } + } else if config.AccessKey != "" { chain = []credentials.Provider{wrapCredentialsProvider(&credentials.Static{ Value: credentials.Value{ AccessKeyID: config.AccessKey, diff --git a/pkg/objstore/s3/s3_aws_sdk_auth.go b/pkg/objstore/s3/s3_aws_sdk_auth.go new file mode 100644 index 0000000000..393a931d17 --- /dev/null +++ b/pkg/objstore/s3/s3_aws_sdk_auth.go @@ -0,0 +1,54 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package s3 + +import ( + "context" + + aws "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/pkg/errors" +) + +// AWSSDKAuth retrieves credentials from the aws-sdk-go. +type AWSSDKAuth struct { + Region string + creds aws.Credentials +} + +// NewAWSSDKAuth returns a pointer to a new Credentials object +// wrapping the environment variable provider. +func NewAWSSDKAuth(region string) *credentials.Credentials { + return credentials.New(&AWSSDKAuth{ + Region: region, + }) +} + +// Retrieve retrieves the keys from the environment. +func (a *AWSSDKAuth) Retrieve() (credentials.Value, error) { + cfg, err := awsconfig.LoadDefaultConfig(context.TODO(), awsconfig.WithRegion(a.Region)) + if err != nil { + return credentials.Value{}, errors.Wrap(err, "load AWS SDK config") + } + + creds, err := cfg.Credentials.Retrieve(context.TODO()) + if err != nil { + return credentials.Value{}, errors.Wrap(err, "retrieve AWS SDK credentials") + } + + a.creds = creds + + return credentials.Value{ + AccessKeyID: creds.AccessKeyID, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + SignerType: credentials.SignatureV4, + }, nil +} + +// IsExpired returns if the credentials have been retrieved. +func (a *AWSSDKAuth) IsExpired() bool { + return a.creds.Expired() +} From 3b669dce1eaffe5c771c6459c8e85c4a90392a6a Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Thu, 3 Feb 2022 10:08:52 +0100 Subject: [PATCH 2/4] Upgrade github.com/aws/aws-sdk-go-v2 Signed-off-by: Sylvain Rabot --- go.mod | 20 +++++++++++--------- go.sum | 41 ++++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index a573359e54..a5edb003cb 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a github.com/alicebob/miniredis/v2 v2.14.3 github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible - github.com/aws/aws-sdk-go-v2 v1.11.1 - github.com/aws/aws-sdk-go-v2/config v1.8.1 + github.com/aws/aws-sdk-go-v2 v1.13.0 + github.com/aws/aws-sdk-go-v2/config v1.13.1 github.com/baidubce/bce-sdk-go v0.9.81 github.com/blang/semver/v4 v4.0.0 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b @@ -107,13 +107,15 @@ require ( github.com/armon/go-radix v1.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/aws/aws-sdk-go v1.42.8 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.4.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect - github.com/aws/smithy-go v1.9.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 // indirect + github.com/aws/smithy-go v1.10.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 // indirect diff --git a/go.sum b/go.sum index 89e39d09e9..afe1d12b73 100644 --- a/go.sum +++ b/go.sum @@ -241,28 +241,31 @@ github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go v1.42.8 h1:Tj2RP4Fas1mYchwbmw0qWLJIEATAseyp5iTa1D+LWYQ= github.com/aws/aws-sdk-go v1.42.8/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.11.1 h1:GzvOVAdTbWxhEMRK4FfiblkGverOkAT0UodDxC1jHQM= -github.com/aws/aws-sdk-go-v2 v1.11.1/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= -github.com/aws/aws-sdk-go-v2/config v1.8.1 h1:AcAenV2NVwOViG+3ts73uT08L1olN4NBNNz7lUlHSUo= -github.com/aws/aws-sdk-go-v2/config v1.8.1/go.mod h1:AQtpYfVYjuuft4Dgh0jGSkPQJ9MvmK9vXfSub7oSXlI= -github.com/aws/aws-sdk-go-v2/credentials v1.4.1 h1:oDiUP50hKRwC6xAgESAj46lgL2prJRZQWnCBzn+TU/c= -github.com/aws/aws-sdk-go-v2/credentials v1.4.1/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 h1:OxTAgH8Y4BXHD6PGCJ8DHx2kaZPCQfSTqmDsdRZFezE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y= -github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 h1:d95cddM3yTm4qffj3P6EnP+TzX1SSkWaQypXSgT/hpA= -github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw= +github.com/aws/aws-sdk-go-v2 v1.13.0 h1:1XIXAfxsEmbhbj5ry3D3vX+6ZcUYvIqSm4CWWEuGZCA= +github.com/aws/aws-sdk-go-v2 v1.13.0/go.mod h1:L6+ZpqHaLbAaxsqV0L4cvxZY7QupWJB4fhkf8LXvC7w= +github.com/aws/aws-sdk-go-v2/config v1.13.1 h1:yLv8bfNoT4r+UvUKQKqRtdnvuWGMK5a82l4ru9Jvnuo= +github.com/aws/aws-sdk-go-v2/config v1.13.1/go.mod h1:Ba5Z4yL/UGbjQUzsiaN378YobhFo0MLfueXGiOsYtEs= +github.com/aws/aws-sdk-go-v2/credentials v1.8.0 h1:8Ow0WcyDesGNL0No11jcgb1JAtE+WtubqXjgxau+S0o= +github.com/aws/aws-sdk-go-v2/credentials v1.8.0/go.mod h1:gnMo58Vwx3Mu7hj1wpcG8DI0s57c9o42UQ6wgTQT5to= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 h1:NITDuUZO34mqtOwFWZiXo7yAHj7kf+XPE+EiKuCBNUI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0/go.mod h1:I6/fHT/fH460v09eg2gVrd8B/IqskhNdpcLH0WNO3QI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 h1:CRiQJ4E2RhfDdqbie1ZYDo8QtIo75Mk7oTdJSfwJTMQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4/go.mod h1:XHgQ7Hz2WY2GAn//UXHofLfPXWh+s62MbMOijrg12Lw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 h1:3ADoioDMOtF4uiK59vCpplpCwugEU+v4ZFD29jDL3RQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0/go.mod h1:BsCSJHx5DnDXIrOcqB8KN1/B+hXLG/bi4Y6Vjcx/x9E= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 h1:ixotxbfTCFpqbuwFv/RcZwyzhkxPSYDYEMcj4niB5Uk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5/go.mod h1:R3sWUqPcfXSiF/LSFJhjyJmpg9uV6yP2yv3YZZjldVI= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= -github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg= -github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= -github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU= -github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 h1:4QAOB3KrvI1ApJK14sliGr3Ie2pjyvNypn/lfzDHfUw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0/go.mod h1:K/qPe6AP2TGYv4l6n7c88zh9jWBDf6nHhvg1fx/EWfU= +github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 h1:1qLJeQGBmNQW3mBNzK2CFmrQNmoXWrscPqsrAaU1aTA= +github.com/aws/aws-sdk-go-v2/service/sso v1.9.0/go.mod h1:vCV4glupK3tR7pw7ks7Y4jYRL86VvxS+g5qk04YeWrU= +github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 h1:ksiDXhvNYg0D2/UFkLejsaz3LqpW5yjNQ8Nx9Sn2c0E= +github.com/aws/aws-sdk-go-v2/service/sts v1.14.0/go.mod h1:u0xMJKDvvfocRjiozsoZglVNXRG19043xzp3r2ivLIk= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= -github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.10.0 h1:gsoZQMNHnX+PaghNw4ynPsyGP7aUCqx5sY2dlPQsZ0w= +github.com/aws/smithy-go v1.10.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/baidubce/bce-sdk-go v0.9.81 h1:n8KfThLG9fvGv3A+RtTt/jKhg/FPPRpo+iNnS2r+iPI= github.com/baidubce/bce-sdk-go v0.9.81/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= From 6b6a39f2a6073c44a434728dc9e958b1da333dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Thu, 3 Feb 2022 11:22:59 +0200 Subject: [PATCH 3/4] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7d8319ac2..db2c4cda9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased -- [#4667](https://github.com/thanos-io/thanos/pull/4667) Add a pure aws-sdk auth for s3 storage. - ### Added - [#5110](https://github.com/thanos-io/thanos/pull/5110) Block: Do not upload DebugMeta files to obj store. @@ -27,6 +25,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4974](https://github.com/thanos-io/thanos/pull/4974) Store: Support tls_config configuration for connecting with Azure storage. - [#4999](https://github.com/thanos-io/thanos/pull/4999) COS: Support `endpoint` configuration for vpc internal endpoint. - [#5059](https://github.com/thanos-io/thanos/pull/5059) Compactor: Adding minimum retention flag validation for downsampling retention. +- [#4667](https://github.com/thanos-io/thanos/pull/4667) Add a pure aws-sdk auth for s3 storage. ### Fixed From 7746efdf8de6ec43f24958e7d6285ca30ec2ff8c Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Fri, 4 Feb 2022 15:55:25 +0100 Subject: [PATCH 4/4] Move s3 conf test in validate() Signed-off-by: Sylvain Rabot --- pkg/objstore/s3/s3.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index a47d0113b8..1ece1a51f8 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -226,10 +226,6 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return nil, err } - if config.AWSSDKAuth && config.AccessKey != "" { - return nil, errors.New("aws_sdk_auth and access_key are mutually exclusive configurations") - } - if config.AWSSDKAuth { chain = []credentials.Provider{ wrapCredentialsProvider(&AWSSDKAuth{Region: config.Region}), @@ -346,6 +342,10 @@ func validate(conf Config) error { return errors.New("no s3 endpoint in config file") } + if conf.AWSSDKAuth && conf.AccessKey != "" { + return errors.New("aws_sdk_auth and access_key are mutually exclusive configurations") + } + if conf.AccessKey == "" && conf.SecretKey != "" { return errors.New("no s3 access_key specified while secret_key is present in config file; either both should be present in config or envvars/IAM should be used.") }