From f7eb70a9a817404a9a44daa1c88424773abdf740 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 13 Oct 2021 12:41:51 +0300 Subject: [PATCH 01/11] added assume role fetch workflow --- .github/workflows/cq_fetch.yml | 82 +++++++++++++++++++++++++++++++++ client/testdata/assume_role.hcl | 27 +++++++++++ 2 files changed, 109 insertions(+) create mode 100644 client/testdata/assume_role.hcl diff --git a/.github/workflows/cq_fetch.yml b/.github/workflows/cq_fetch.yml index c665ae24f..26f41ada6 100644 --- a/.github/workflows/cq_fetch.yml +++ b/.github/workflows/cq_fetch.yml @@ -83,4 +83,86 @@ jobs: SLACK_COLOR: ${{ job.status }} SLACK_MESSAGE: 'AWS - nightly fetch failed' SLACK_TITLE: AWS - nightly fetch failed + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + nightly_assume_role_fetch: + if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'cloudquery/cq-provider-aws') + strategy: + matrix: + dbversion: [ "postgres:latest" ] + go: [ "1.17" ] + platform: [ ubuntu-latest ] # can not run in macOS and widnowsOS + runs-on: ${{ matrix.platform }} + services: + postgres: + image: ${{ matrix.dbversion }} + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.17 + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + go get -v -t -d ./... + + - name: Build + run: go build -v . + + - name: Cache CQ + id: cache-cq-binary + uses: actions/cache@v2 + with: + path: cloudquery + key: ${{ runner.os }}-${{ hashFiles('cloudquery') }} + + - name: Download Cloudquery + if: steps.cache-cq-binary.cache-hit != 'true' + run: | + curl -L https://github.com/cloudquery/cloudquery/releases/latest/download/cloudquery_${OS}_x86_64 -o cloudquery + chmod a+x cloudquery + env: + OS: Linux + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.ASSUME_ROLE_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.ASSUME_ROLE_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-east-1 + + - name: Fetch + run: | + rm -rf .cq_reattach + CQ_PROVIDER_DEBUG=1 go run main.go & while [ ! -f .cq_reattach ]; do sleep 1; done && ./cloudquery fetch --config ./client/testdata/nightly.hcl --enable-console-log --fail-on-error + env: + CQ_REATTACH_PROVIDERS: .cq_reattach + CQ_ASSUME_ROLE_ACCOUNT_ID: ${{ secrets.CQ_ASSUME_ROLE_ACCOUNT_ID }} + CQ_ASSUME_ROLE_ARN: ${{ secrets.CQ_ASSUME_ROLE_ARN }} + + - name: Slack Notification + uses: rtCamp/action-slack-notify@v2 + if: ${{ failure() }} + env: + SLACK_CHANNEL: oss-tests + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: 'AWS - nightly(AssumeRole) fetch failed' + SLACK_TITLE: AWS - nightly(AssumeRole) fetch failed SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} \ No newline at end of file diff --git a/client/testdata/assume_role.hcl b/client/testdata/assume_role.hcl new file mode 100644 index 000000000..021888cae --- /dev/null +++ b/client/testdata/assume_role.hcl @@ -0,0 +1,27 @@ +cloudquery { + plugin_directory = "./cq/providers" + policy_directory = "./cq/policies" + + provider "aws" { + source = "cloudquery/cq-provider-aws" + version = "latest" + } + + connection { + dsn = "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable" + } +} + +provider "aws" { + configuration { + + accounts "${assume_account_id}"{ + role_arn = "${assume_role_arn}" + } + + max_retries = 7 + max_backoff = 60 + } + resources = [ + "*"] +} From 01aa44a648926374ccea59ade16d43f394c6f253 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 08:28:03 +0200 Subject: [PATCH 02/11] fixed assume role fetch --- .github/workflows/cq_fetch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cq_fetch.yml b/.github/workflows/cq_fetch.yml index 26f41ada6..253a9f7ad 100644 --- a/.github/workflows/cq_fetch.yml +++ b/.github/workflows/cq_fetch.yml @@ -145,7 +145,6 @@ jobs: with: aws-access-key-id: ${{ secrets.ASSUME_ROLE_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.ASSUME_ROLE_AWS_SECRET_ACCESS_KEY }} - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-east-1 - name: Fetch From 860865bbd1b2626f06877438d8fee0181208e154 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 08:51:02 +0200 Subject: [PATCH 03/11] updated cq env vars --- client/testdata/assume_role.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/testdata/assume_role.hcl b/client/testdata/assume_role.hcl index 021888cae..78f27bbd9 100644 --- a/client/testdata/assume_role.hcl +++ b/client/testdata/assume_role.hcl @@ -15,8 +15,8 @@ cloudquery { provider "aws" { configuration { - accounts "${assume_account_id}"{ - role_arn = "${assume_role_arn}" + accounts "${ASSUME_ROLE_ACCOUNT_ID}"{ + role_arn = "${ASSUME_ROLE_ARN}" } max_retries = 7 From 9976becb7a833fb9543b9e873dff434fd26cb643 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 09:04:06 +0200 Subject: [PATCH 04/11] fixed --- .github/workflows/cq_fetch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cq_fetch.yml b/.github/workflows/cq_fetch.yml index 253a9f7ad..74284794d 100644 --- a/.github/workflows/cq_fetch.yml +++ b/.github/workflows/cq_fetch.yml @@ -122,6 +122,7 @@ jobs: run: | go get -v -t -d ./... + - name: Build run: go build -v . @@ -150,7 +151,7 @@ jobs: - name: Fetch run: | rm -rf .cq_reattach - CQ_PROVIDER_DEBUG=1 go run main.go & while [ ! -f .cq_reattach ]; do sleep 1; done && ./cloudquery fetch --config ./client/testdata/nightly.hcl --enable-console-log --fail-on-error + CQ_PROVIDER_DEBUG=1 go run main.go & while [ ! -f .cq_reattach ]; do sleep 1; done && ./cloudquery fetch --config ./client/testdata/assume_role.hcl --enable-console-log --fail-on-error env: CQ_REATTACH_PROVIDERS: .cq_reattach CQ_ASSUME_ROLE_ACCOUNT_ID: ${{ secrets.CQ_ASSUME_ROLE_ACCOUNT_ID }} From 469cbf6c775ba4d8594f78f36688392b4383e9da Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 09:19:15 +0200 Subject: [PATCH 05/11] fixed --- .github/workflows/cq_fetch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cq_fetch.yml b/.github/workflows/cq_fetch.yml index 74284794d..bf9d97edc 100644 --- a/.github/workflows/cq_fetch.yml +++ b/.github/workflows/cq_fetch.yml @@ -154,8 +154,8 @@ jobs: CQ_PROVIDER_DEBUG=1 go run main.go & while [ ! -f .cq_reattach ]; do sleep 1; done && ./cloudquery fetch --config ./client/testdata/assume_role.hcl --enable-console-log --fail-on-error env: CQ_REATTACH_PROVIDERS: .cq_reattach - CQ_ASSUME_ROLE_ACCOUNT_ID: ${{ secrets.CQ_ASSUME_ROLE_ACCOUNT_ID }} - CQ_ASSUME_ROLE_ARN: ${{ secrets.CQ_ASSUME_ROLE_ARN }} + CQ_VAR_ASSUME_ROLE_ACCOUNT_ID: ${{ secrets.CQ_ASSUME_ROLE_ACCOUNT_ID }} + CQ_VAR_ASSUME_ROLE_ARN: ${{ secrets.CQ_ASSUME_ROLE_ARN }} - name: Slack Notification uses: rtCamp/action-slack-notify@v2 From d3b4317104a08d5c7bc4cfb1b629e5dda7b2fe39 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 09:47:47 +0200 Subject: [PATCH 06/11] fixed --- client/testdata/assume_role.hcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/testdata/assume_role.hcl b/client/testdata/assume_role.hcl index 78f27bbd9..532624313 100644 --- a/client/testdata/assume_role.hcl +++ b/client/testdata/assume_role.hcl @@ -15,8 +15,9 @@ cloudquery { provider "aws" { configuration { - accounts "${ASSUME_ROLE_ACCOUNT_ID}"{ + accounts "default" { role_arn = "${ASSUME_ROLE_ARN}" + external_id = "${ASSUME_ROLE_ACCOUNT_ID}" } max_retries = 7 From bc100c5dc52624f688380f0e2b2dcca7409e6740 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 09:56:08 +0200 Subject: [PATCH 07/11] added account id field to account block --- client/client.go | 30 +++++++++++++++++++----------- client/config.go | 1 + client/testdata/assume_role.hcl | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/client/client.go b/client/client.go index 265e449e1..0d8f1eb77 100644 --- a/client/client.go +++ b/client/client.go @@ -236,18 +236,26 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe if len(awsConfig.Accounts) == 0 { awsConfig.Accounts = append(awsConfig.Accounts, Account{ - ID: "default", - RoleARN: "default", + ID: "default", + AccountID: "default", + RoleARN: "default", }) } for _, account := range awsConfig.Accounts { var err error var awsCfg aws.Config + + var accountID = account.AccountID + + if accountID == "" { + accountID = account.ID + } + // This is a try to solve https://aws.amazon.com/premiumsupport/knowledge-center/iam-validate-access-credentials/ // with this https://github.com/aws/aws-sdk-go-v2/issues/515#issuecomment-607387352 switch { - case account.ID != "default" && account.RoleARN != "": + case accountID != "default" && account.RoleARN != "": // assume role if specified (SDK takes it from default or env var: AWS_PROFILE) awsCfg, err = config.LoadDefaultConfig( ctx, @@ -255,7 +263,7 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe config.WithRetryer(newRetryer(awsConfig.MaxRetries, awsConfig.MaxBackoff)), ) if err != nil { - return nil, fmt.Errorf(awsFailedToConfigureErrMsg, account.ID, err) + return nil, fmt.Errorf(awsFailedToConfigureErrMsg, accountID, err) } opts := make([]func(*stscreds.AssumeRoleOptions), 0, 1) if account.ExternalID != "" { @@ -265,11 +273,11 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe } provider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(awsCfg), account.RoleARN, opts...) awsCfg.Credentials = aws.NewCredentialsCache(provider) - case account.ID != "default": + case accountID != "default": awsCfg, err = config.LoadDefaultConfig( ctx, config.WithDefaultRegion(defaultRegion), - config.WithSharedConfigProfile(account.ID), + config.WithSharedConfigProfile(accountID), config.WithRetryer(newRetryer(awsConfig.MaxRetries, awsConfig.MaxBackoff)), ) default: @@ -281,19 +289,19 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe } if err != nil { - return nil, fmt.Errorf(awsFailedToConfigureErrMsg, account.ID, err) + return nil, fmt.Errorf(awsFailedToConfigureErrMsg, accountID, err) } if awsConfig.AWSDebug { awsCfg.ClientLogMode = aws.LogRequest | aws.LogResponse | aws.LogRetries - awsCfg.Logger = AwsLogger{logger.With("account", obfuscateAccountId(account.ID))} + awsCfg.Logger = AwsLogger{logger.With("account", obfuscateAccountId(accountID))} } svc := sts.NewFromConfig(awsCfg) output, err := svc.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}, func(o *sts.Options) { o.Region = "aws-global" }) if err != nil { - return nil, fmt.Errorf(awsFailedToConfigureErrMsg, account.ID, err) + return nil, fmt.Errorf(awsFailedToConfigureErrMsg, accountID, err) } // This is a work-around to skip disabled regions // https://github.com/aws/aws-sdk-go-v2/issues/1068 @@ -303,12 +311,12 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe o.Region = "us-east-1" }) if err != nil { - return nil, fmt.Errorf("failed to find disabled regions for account %s. AWS Error: %w", account.ID, err) + return nil, fmt.Errorf("failed to find disabled regions for account %s. AWS Error: %w", accountID, err) } client.regions = filterDisabledRegions(client.regions, res.Regions) if len(client.regions) == 0 { - return nil, fmt.Errorf("no enabled regions provided in config for account %s", account.ID) + return nil, fmt.Errorf("no enabled regions provided in config for account %s", accountID) } if client.AccountID == "" { diff --git a/client/config.go b/client/config.go index aa1c008b8..6b9270607 100644 --- a/client/config.go +++ b/client/config.go @@ -2,6 +2,7 @@ package client type Account struct { ID string `hcl:",label"` + AccountID string `hcl:"account_id,optional"` RoleARN string `hcl:"role_arn,optional"` ExternalID string `hcl:"external_id,optional"` } diff --git a/client/testdata/assume_role.hcl b/client/testdata/assume_role.hcl index 532624313..ee2013038 100644 --- a/client/testdata/assume_role.hcl +++ b/client/testdata/assume_role.hcl @@ -17,7 +17,7 @@ provider "aws" { accounts "default" { role_arn = "${ASSUME_ROLE_ARN}" - external_id = "${ASSUME_ROLE_ACCOUNT_ID}" + account_id = "${ASSUME_ROLE_ACCOUNT_ID}" } max_retries = 7 From 0f53b5856224bcf680a011261ad2e13514dd0f69 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 10:11:50 +0200 Subject: [PATCH 08/11] increased max retries --- client/testdata/assume_role.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/testdata/assume_role.hcl b/client/testdata/assume_role.hcl index ee2013038..1ed7fde56 100644 --- a/client/testdata/assume_role.hcl +++ b/client/testdata/assume_role.hcl @@ -20,7 +20,7 @@ provider "aws" { account_id = "${ASSUME_ROLE_ACCOUNT_ID}" } - max_retries = 7 + max_retries = 20 max_backoff = 60 } resources = [ From e0a306898469afead24f2282d66a679523795882 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 11:40:35 +0200 Subject: [PATCH 09/11] fixed CR --- client/client.go | 7 ++++--- client/config.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/client/client.go b/client/client.go index 0d8f1eb77..20fa53657 100644 --- a/client/client.go +++ b/client/client.go @@ -84,6 +84,7 @@ var allRegions = []string{ const ( defaultRegion = "us-east-1" awsFailedToConfigureErrMsg = "failed to configure provider for account %s. AWS Error: %w" + defaultVar = "default" ) type Services struct { @@ -236,9 +237,9 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe if len(awsConfig.Accounts) == 0 { awsConfig.Accounts = append(awsConfig.Accounts, Account{ - ID: "default", - AccountID: "default", - RoleARN: "default", + ID: defaultVar, + AccountID: defaultVar, + RoleARN: defaultVar, }) } diff --git a/client/config.go b/client/config.go index 6b9270607..59e75f560 100644 --- a/client/config.go +++ b/client/config.go @@ -18,9 +18,10 @@ type Config struct { func (c Config) Example() string { return `configuration { // Optional. if you want to assume role to multiple account and fetch data from them - //accounts "" { + //accounts "" { // Optional. Role ARN we want to assume when accessing this account // role_arn = + // account_id = //optional // } // Optional. by default assumes all regions // regions = ["us-east-1", "us-west-2"] From cc0bd857a14a1672cb12bc3d81bef19c10747d78 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 11:42:56 +0200 Subject: [PATCH 10/11] fixed CR --- client/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index 20fa53657..a9265be64 100644 --- a/client/client.go +++ b/client/client.go @@ -247,8 +247,9 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe var err error var awsCfg aws.Config + // account id can be defined in account block label or in block attr + // we take the block att as default and use the block label if the attr is not defined var accountID = account.AccountID - if accountID == "" { accountID = account.ID } From 123b9598c0dc5d250f3f5895f5104ca472ff1792 Mon Sep 17 00:00:00 2001 From: Itay Zagron Date: Wed, 20 Oct 2021 14:06:09 +0200 Subject: [PATCH 11/11] fixed CR --- client/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/config.go b/client/config.go index 59e75f560..7285beeb1 100644 --- a/client/config.go +++ b/client/config.go @@ -21,7 +21,8 @@ func (c Config) Example() string { //accounts "" { // Optional. Role ARN we want to assume when accessing this account // role_arn = - // account_id = //optional + // Optional. Account ID we want to assume when accessing this account - override the block label + // account_id = // } // Optional. by default assumes all regions // regions = ["us-east-1", "us-west-2"]