diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 652cc13351f3..a47173644994 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -65,6 +65,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220] - Added Airflow lightweight module {pull}26220[26220] - Add state_job metricset to Kubernetes module{pull}26479[26479] +- Bump AWS SDK version to v0.24.0 for WebIdentity authentication flow {issue}19393[19393] {pull}27126[27126] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 92b117280597..185b995e19ab 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2860,11 +2860,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-lambda-go@v1.6. -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2 -Version: v0.9.0 +Version: v0.24.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2@v0.9.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2@v0.24.0/LICENSE.txt: Apache License @@ -9069,11 +9069,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/go-sql-driver/mysql -Version: v1.4.1 +Version: v1.5.0 Licence type (autodetected): MPL-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-sql-driver/mysql@v1.4.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/go-sql-driver/mysql@v1.5.0/LICENSE: Mozilla Public License Version 2.0 ================================== diff --git a/go.mod b/go.mod index 683643f5f77e..bd7ea03215b1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/aws/aws-lambda-go v1.6.0 - github.com/aws/aws-sdk-go-v2 v0.9.0 + github.com/aws/aws-sdk-go-v2 v0.24.0 github.com/awslabs/goformation/v4 v4.1.0 github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible @@ -81,7 +81,7 @@ require ( github.com/fsnotify/fsnotify v1.4.9 github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 // indirect github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect - github.com/go-sql-driver/mysql v1.4.1 + github.com/go-sql-driver/mysql v1.5.0 github.com/go-test/deep v1.0.7 github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e diff --git a/go.sum b/go.sum index dede3be284c6..64da5538d85f 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-lambda-go v1.6.0 h1:T+u/g79zPKw1oJM7xYhvpq7i4Sjc0iVsXZUaqRVVSOg= github.com/aws/aws-lambda-go v1.6.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= -github.com/aws/aws-sdk-go-v2 v0.9.0 h1:dWtJKGRFv3UZkMBQaIzMsF0/y4ge3iQPWTzeC4r/vl4= -github.com/aws/aws-sdk-go-v2 v0.9.0/go.mod h1:sa1GePZ/LfBGI4dSq30f6uR4Tthll8axxtEPvlpXZ8U= +github.com/aws/aws-sdk-go-v2 v0.24.0 h1:R0lL0krk9EyTI1vmO1ycoeceGZotSzCKO51LbPGq3rU= +github.com/aws/aws-sdk-go-v2 v0.24.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw= github.com/awslabs/goformation/v3 v3.1.0/go.mod h1:hQ5RXo3GNm2laHWKizDzU5DsDy+yNcenSca2UxN0850= github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56ubpywGU= github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= @@ -322,8 +322,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= @@ -806,7 +806,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -914,7 +913,6 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= diff --git a/libbeat/tests/resources/goroutines.go b/libbeat/tests/resources/goroutines.go index 3351c7b23772..7ab2c6803ae5 100644 --- a/libbeat/tests/resources/goroutines.go +++ b/libbeat/tests/resources/goroutines.go @@ -26,7 +26,10 @@ import ( "time" ) -const defaultFinalizationTimeout = 5 * time.Second +// This is the maximum waiting time for goroutine shutdown. +// If the shutdown happens earlier the waiting time will be lower. +// High maximum waiting time was due to flaky tests on CI workers +const defaultFinalizationTimeout = 35 * time.Second // GoroutinesChecker keeps the count of goroutines when it was created // so later it can check if this number has increased diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 7d23de407fa7..b7f2991e68b3 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -304,6 +304,45 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo [id="aws-credentials-config"] include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] +[float] +== Running on EKS + +* *WebIdentity authentication flow* + +See documentation in order to create a IAM Role for Service account: +https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + +Once you have create the IRSA you can annotate `metricbeat` service account with it +[source,yaml] +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + +In order to enable WebIdentity authentication flow you need to add a trust relationship +to the IRSA: +[source,json] + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.REGION.amazonaws.com/id/:sub": "system:serviceaccount:kube-system:metricbeat", + "oidc.eks.REGION.amazonaws.com/id/:aud": "sts.amazonaws.com" + } + } + } + +In this case there's no need to add `role_arn` to modules config. + [float] === Example configuration diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go index 80ee9b31825e..2e44d35b3071 100644 --- a/x-pack/filebeat/input/awscloudwatch/input.go +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -12,7 +12,6 @@ import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" - "github.com/aws/aws-sdk-go-v2/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/cloudwatchlogsiface" "github.com/pkg/errors" @@ -167,7 +166,8 @@ func (in *awsCloudWatchInput) run() { for in.inputCtx.Err() == nil { err := in.getLogEventsFromCloudWatch(svc) if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + var aerr *awssdk.RequestCanceledError + if errors.As(err, &aerr) { continue } in.logger.Error("getLogEventsFromCloudWatch failed: ", err) diff --git a/x-pack/filebeat/input/awss3/collector.go b/x-pack/filebeat/input/awss3/collector.go index e4702a6fd011..726f3b691230 100644 --- a/x-pack/filebeat/input/awss3/collector.go +++ b/x-pack/filebeat/input/awss3/collector.go @@ -11,6 +11,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -100,7 +101,8 @@ func (c *s3Collector) run() { // receive messages from sqs output, err := c.receiveMessage(c.sqs, c.visibilityTimeout) if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + var aerr *awssdk.RequestCanceledError + if errors.As(err, &aerr) { continue } c.logger.Error("SQS ReceiveMessageRequest failed: ", err) @@ -365,14 +367,13 @@ func (c *s3Collector) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, resp, err := req.Send(ctx) if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - // If the SDK can determine the request or retry delay was canceled - // by a context the ErrCodeRequestCanceled error will be returned. - if awsErr.Code() == awssdk.ErrCodeRequestCanceled { - c.logger.Error(fmt.Errorf("s3 GetObjectRequest canceled for '%s' from S3 bucket '%s': %w", info.key, info.name, err)) - return err - } + var aerr *awssdk.RequestCanceledError + if errors.As(err, &aerr) { + c.logger.Error(fmt.Errorf("s3 GetObjectRequest canceled for '%s' from S3 bucket '%s': %w", info.key, info.name, err)) + return err + } + if awsErr, ok := err.(awserr.Error); ok { if awsErr.Code() == "NoSuchKey" { c.logger.Warnf("Cannot find s3 file '%s' from S3 bucket '%s'", info.key, info.name) return nil @@ -579,7 +580,8 @@ func (c *s3Collector) deleteMessage(queueURL string, messagesReceiptHandle strin _, err := req.Send(ctx) if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + var aerr *awssdk.RequestCanceledError + if errors.As(err, &aerr) { return nil } return fmt.Errorf("SQS DeleteMessageRequest failed: %w", err) diff --git a/x-pack/filebeat/input/awss3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go index 97953e39bdf5..5e127839f40e 100644 --- a/x-pack/filebeat/input/awss3/collector_test.go +++ b/x-pack/filebeat/input/awss3/collector_test.go @@ -57,6 +57,7 @@ func (m *MockS3Client) GetObjectRequest(input *s3.GetObjectInput) s3.GetObjectRe Body: logBody, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } diff --git a/x-pack/functionbeat/manager/aws/cli_manager.go b/x-pack/functionbeat/manager/aws/cli_manager.go index e5e35bbbdf2e..a4b4b7dbf3de 100644 --- a/x-pack/functionbeat/manager/aws/cli_manager.go +++ b/x-pack/functionbeat/manager/aws/cli_manager.go @@ -5,6 +5,7 @@ package aws import ( + "context" "fmt" "io/ioutil" "os" @@ -68,7 +69,7 @@ func (c *CLIManager) deployTemplate(update bool, name string) error { c.log.Debugf("Using cloudformation template:\n%s", templateData.json) - _, err = c.awsCfg.Credentials.Retrieve() + _, err = c.awsCfg.Credentials.Retrieve(context.Background()) if err != nil { return fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %+v", err) } @@ -150,7 +151,7 @@ func (c *CLIManager) Remove(name string) error { c.log.Debugf("Removing function: %s", name) defer c.log.Debugf("Removal of function '%s' complete", name) - _, err := c.awsCfg.Credentials.Retrieve() + _, err := c.awsCfg.Credentials.Retrieve(context.Background()) if err != nil { return fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %+v", err) } diff --git a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go index a81ef707aba9..a88aff411fc0 100644 --- a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go +++ b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go @@ -55,7 +55,7 @@ func (m *mockCloudFormationClient) DescribeStackEventsRequest( }() httpReq, _ := http.NewRequest("", "", nil) return cloudformation.DescribeStackEventsRequest{ - Request: &aws.Request{Data: &m.Responses[m.Index], HTTPRequest: httpReq}, + Request: &aws.Request{Data: &m.Responses[m.Index], HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } diff --git a/x-pack/functionbeat/manager/aws/op_cloudformation_test.go b/x-pack/functionbeat/manager/aws/op_cloudformation_test.go index 5fd56bd217d9..4533752063bc 100644 --- a/x-pack/functionbeat/manager/aws/op_cloudformation_test.go +++ b/x-pack/functionbeat/manager/aws/op_cloudformation_test.go @@ -44,12 +44,12 @@ func (m *mockCloudformationStack) CreateStackRequest( httpReq, _ := http.NewRequest("", "", nil) if m.err != nil { return cloudformation.CreateStackRequest{ - Request: &aws.Request{Data: m.respCreateStackOutput, Error: m.err, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respCreateStackOutput, Error: m.err, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } return cloudformation.CreateStackRequest{ - Request: &aws.Request{Data: m.respCreateStackOutput, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respCreateStackOutput, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } @@ -63,12 +63,12 @@ func (m *mockCloudformationStack) DeleteStackRequest( httpReq, _ := http.NewRequest("", "", nil) if m.err != nil { return cloudformation.DeleteStackRequest{ - Request: &aws.Request{Data: m.respDeleteStackOutput, Error: m.err, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respDeleteStackOutput, Error: m.err, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } return cloudformation.DeleteStackRequest{ - Request: &aws.Request{Data: m.respDeleteStackOutput, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respDeleteStackOutput, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } @@ -82,12 +82,12 @@ func (m *mockCloudformationStack) DescribeStacksRequest( httpReq, _ := http.NewRequest("", "", nil) if m.err != nil { return cloudformation.DescribeStacksRequest{ - Request: &aws.Request{Data: m.respDescribeStacksOutput, Error: m.err, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respDescribeStacksOutput, Error: m.err, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } return cloudformation.DescribeStacksRequest{ - Request: &aws.Request{Data: m.respDescribeStacksOutput, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respDescribeStacksOutput, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } @@ -101,12 +101,12 @@ func (m *mockCloudformationStack) UpdateStackRequest( httpReq, _ := http.NewRequest("", "", nil) if m.err != nil { return cloudformation.UpdateStackRequest{ - Request: &aws.Request{Data: m.respUpdateStackOutput, Error: m.err, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respUpdateStackOutput, Error: m.err, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } return cloudformation.UpdateStackRequest{ - Request: &aws.Request{Data: m.respUpdateStackOutput, HTTPRequest: httpReq}, + Request: &aws.Request{Data: m.respUpdateStackOutput, HTTPRequest: httpReq, Retryer: aws.NoOpRetryer{}}, } } diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index cae5cb1b6b51..662da9d570a6 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -56,6 +56,7 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) { if config.AccessKeyID != "" || config.SecretAccessKey != "" || config.SessionToken != "" { return getAccessKeys(config), nil } + return getSharedCredentialProfile(config) } @@ -84,7 +85,6 @@ func getAccessKeys(config ConfigAWS) awssdk.Config { return getRoleArn(config, awsConfig) } - logger.Debug("Using access keys for AWS credential") return awsConfig } diff --git a/x-pack/libbeat/common/aws/credentials_test.go b/x-pack/libbeat/common/aws/credentials_test.go index 52b8f1433627..2d9d6e9664c6 100644 --- a/x-pack/libbeat/common/aws/credentials_test.go +++ b/x-pack/libbeat/common/aws/credentials_test.go @@ -5,6 +5,7 @@ package aws import ( + "context" "testing" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -20,7 +21,7 @@ func TestGetAWSCredentials(t *testing.T) { awsConfig, err := GetAWSCredentials(inputConfig) assert.NoError(t, err) - retrievedAWSConfig, err := awsConfig.Credentials.Retrieve() + retrievedAWSConfig, err := awsConfig.Credentials.Retrieve(context.Background()) assert.NoError(t, err) assert.Equal(t, inputConfig.AccessKeyID, retrievedAWSConfig.AccessKeyID) diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index df18966b2af6..2f5d5f442108 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -295,3 +295,42 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo [id="aws-credentials-config"] include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] + +[float] +== Running on EKS + +* *WebIdentity authentication flow* + +See documentation in order to create a IAM Role for Service account: +https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + +Once you have create the IRSA you can annotate `metricbeat` service account with it +[source,yaml] +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + +In order to enable WebIdentity authentication flow you need to add a trust relationship +to the IRSA: +[source,json] + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.REGION.amazonaws.com/id/:sub": "system:serviceaccount:kube-system:metricbeat", + "oidc.eks.REGION.amazonaws.com/id/:aud": "sts.amazonaws.com" + } + } + } + +In this case there's no need to add `role_arn` to modules config. diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f22a1b5ffe1d..0b59f9ebc7fc 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -81,7 +81,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err) } - _, err = awsConfig.Credentials.Retrieve() + _, err = awsConfig.Credentials.Retrieve(context.Background()) if err != nil { return nil, fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %w", err) } diff --git a/x-pack/metricbeat/module/aws/aws_test.go b/x-pack/metricbeat/module/aws/aws_test.go index 398623f67852..7d89adca5740 100644 --- a/x-pack/metricbeat/module/aws/aws_test.go +++ b/x-pack/metricbeat/module/aws/aws_test.go @@ -38,6 +38,7 @@ func (m *MockEC2Client) DescribeRegionsRequest(input *ec2.DescribeRegionsInput) }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 0cc4d0e9511e..29b33dd6137d 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1233,6 +1233,7 @@ func (m *MockCloudWatchClient) ListMetricsRequest(input *cloudwatch.ListMetricsI }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } @@ -1265,6 +1266,7 @@ func (m *MockCloudWatchClient) GetMetricDataRequest(input *cloudwatch.GetMetricD }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } @@ -1282,6 +1284,7 @@ func (m *MockCloudWatchClientWithoutDim) ListMetricsRequest(input *cloudwatch.Li }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } @@ -1314,6 +1317,7 @@ func (m *MockCloudWatchClientWithoutDim) GetMetricDataRequest(input *cloudwatch. }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } @@ -1345,6 +1349,7 @@ func (m *MockResourceGroupsTaggingClient) GetResourcesRequest(input *resourcegro }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index 514a11e4bb03..9cc87efd7480 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -76,6 +76,7 @@ func (m *MockCloudWatchClient) ListMetricsRequest(input *cloudwatch.ListMetricsI }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } @@ -120,6 +121,7 @@ func (m *MockCloudWatchClient) GetMetricDataRequest(input *cloudwatch.GetMetricD }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, } } @@ -167,6 +169,7 @@ func (m *MockResourceGroupsTaggingClient) GetResourcesRequest(input *resourcegro }, }, HTTPRequest: httpReq, + Retryer: awssdk.NoOpRetryer{}, }, Input: input, Copy: m.GetResourcesRequest, @@ -182,6 +185,7 @@ func (m *MockResourceGroupsTaggingClient) GetResourcesRequest(input *resourcegro }, HTTPRequest: httpReq, Operation: op, + Retryer: awssdk.NoOpRetryer{}, }, Input: input, Copy: m.GetResourcesRequest,