Feat/assume role workflow

.github/workflows/cq_fetch.yml

@@ -83,4 +83,86 @@ jobs:
           SLACK_COLOR: ${{ job.status }}
           SLACK_MESSAGE: 'AWS - nightly fetch failed'
           SLACK_TITLE: AWS - nightly fetch failed
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
+  nightly_assume_role_fetch:
+    if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'cloudquery/cq-provider-aws')
+    strategy:
+      matrix:
+        dbversion: [ "postgres:latest" ]
+        go: [ "1.17" ]
+        platform: [ ubuntu-latest ] # can not run in macOS and widnowsOS
+    runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: ${{ matrix.dbversion }}
+        env:
+          POSTGRES_PASSWORD: pass
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.17
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+
+      - name: Build
+        run: go build -v .
+
+      - name: Cache CQ
+        id: cache-cq-binary
+        uses: actions/cache@v2
+        with:
+          path: cloudquery
+          key: ${{ runner.os }}-${{ hashFiles('cloudquery') }}
+
+      - name: Download Cloudquery
+        if: steps.cache-cq-binary.cache-hit != 'true'
+        run: |
+          curl -L https://github.com/cloudquery/cloudquery/releases/latest/download/cloudquery_${OS}_x86_64 -o cloudquery
+          chmod a+x cloudquery
+        env:
+          OS: Linux
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.ASSUME_ROLE_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.ASSUME_ROLE_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Fetch
+        run: |
+          rm -rf .cq_reattach
+          CQ_PROVIDER_DEBUG=1 go run main.go & while [ ! -f .cq_reattach ]; do sleep 1; done && ./cloudquery fetch --config ./client/testdata/assume_role.hcl --enable-console-log --fail-on-error
+        env:
+          CQ_REATTACH_PROVIDERS: .cq_reattach
+          CQ_VAR_ASSUME_ROLE_ACCOUNT_ID: ${{ secrets.CQ_ASSUME_ROLE_ACCOUNT_ID }}
+          CQ_VAR_ASSUME_ROLE_ARN: ${{ secrets.CQ_ASSUME_ROLE_ARN }}
+
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        if: ${{ failure() }}
+        env:
+          SLACK_CHANNEL: oss-tests
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: 'AWS - nightly(AssumeRole) fetch failed'
+          SLACK_TITLE: AWS - nightly(AssumeRole) fetch failed
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

client/client.go

@@ -84,6 +84,7 @@ var allRegions = []string{
 const (
 	defaultRegion              = "us-east-1"
 	awsFailedToConfigureErrMsg = "failed to configure provider for account %s. AWS Error: %w"
+	defaultVar                 = "default"
 )
 
 type Services struct {
@@ -236,26 +237,35 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe
 
 	if len(awsConfig.Accounts) == 0 {
 		awsConfig.Accounts = append(awsConfig.Accounts, Account{
-			ID:      "default",
-			RoleARN: "default",
+			ID:        defaultVar,
+			AccountID: defaultVar,
+			RoleARN:   defaultVar,
 		})
 	}
 
 	for _, account := range awsConfig.Accounts {
 		var err error
 		var awsCfg aws.Config
+
+		// account id can be defined in account block label or in block attr
+		// we take the block att as default and use the block label if the attr is not defined
+		var accountID = account.AccountID
+		if accountID == "" {
+			accountID = account.ID
+		}
+
 		// This is a try to solve https://aws.amazon.com/premiumsupport/knowledge-center/iam-validate-access-credentials/
 		// with this https://github.com/aws/aws-sdk-go-v2/issues/515#issuecomment-607387352
 		switch {
-		case account.ID != "default" && account.RoleARN != "":
+		case accountID != "default" && account.RoleARN != "":
 			// assume role if specified (SDK takes it from default or env var: AWS_PROFILE)
 			awsCfg, err = config.LoadDefaultConfig(
 				ctx,
 				config.WithDefaultRegion(defaultRegion),
 				config.WithRetryer(newRetryer(awsConfig.MaxRetries, awsConfig.MaxBackoff)),
 			)
 			if err != nil {
-				return nil, fmt.Errorf(awsFailedToConfigureErrMsg, account.ID, err)
+				return nil, fmt.Errorf(awsFailedToConfigureErrMsg, accountID, err)
 			}
 			opts := make([]func(*stscreds.AssumeRoleOptions), 0, 1)
 			if account.ExternalID != "" {
@@ -265,11 +275,11 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe
 			}
 			provider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(awsCfg), account.RoleARN, opts...)
 			awsCfg.Credentials = aws.NewCredentialsCache(provider)
-		case account.ID != "default":
+		case accountID != "default":
 			awsCfg, err = config.LoadDefaultConfig(
 				ctx,
 				config.WithDefaultRegion(defaultRegion),
-				config.WithSharedConfigProfile(account.ID),
+				config.WithSharedConfigProfile(accountID),
 				config.WithRetryer(newRetryer(awsConfig.MaxRetries, awsConfig.MaxBackoff)),
 			)
 		default:
@@ -281,19 +291,19 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe
 		}
 
 		if err != nil {
-			return nil, fmt.Errorf(awsFailedToConfigureErrMsg, account.ID, err)
+			return nil, fmt.Errorf(awsFailedToConfigureErrMsg, accountID, err)
 		}
 
 		if awsConfig.AWSDebug {
 			awsCfg.ClientLogMode = aws.LogRequest | aws.LogResponse | aws.LogRetries
-			awsCfg.Logger = AwsLogger{logger.With("account", obfuscateAccountId(account.ID))}
+			awsCfg.Logger = AwsLogger{logger.With("account", obfuscateAccountId(accountID))}
 		}
 		svc := sts.NewFromConfig(awsCfg)
 		output, err := svc.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}, func(o *sts.Options) {
 			o.Region = "aws-global"
 		})
 		if err != nil {
-			return nil, fmt.Errorf(awsFailedToConfigureErrMsg, account.ID, err)
+			return nil, fmt.Errorf(awsFailedToConfigureErrMsg, accountID, err)
 		}
 		// This is a work-around to skip disabled regions
 		// https://github.com/aws/aws-sdk-go-v2/issues/1068
@@ -303,12 +313,12 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe
 				o.Region = "us-east-1"
 			})
 		if err != nil {
-			return nil, fmt.Errorf("failed to find disabled regions for account %s. AWS Error: %w", account.ID, err)
+			return nil, fmt.Errorf("failed to find disabled regions for account %s. AWS Error: %w", accountID, err)
 		}
 		client.regions = filterDisabledRegions(client.regions, res.Regions)
 
 		if len(client.regions) == 0 {
-			return nil, fmt.Errorf("no enabled regions provided in config for account %s", account.ID)
+			return nil, fmt.Errorf("no enabled regions provided in config for account %s", accountID)
 		}
 
 		if client.AccountID == "" {

client/config.go

@@ -2,6 +2,7 @@ package client
 
 type Account struct {
 	ID         string `hcl:",label"`
+	AccountID  string `hcl:"account_id,optional"`
 	RoleARN    string `hcl:"role_arn,optional"`
 	ExternalID string `hcl:"external_id,optional"`
 }
@@ -17,9 +18,11 @@ type Config struct {
 func (c Config) Example() string {
 	return `configuration {
 	// Optional. if you want to assume role to multiple account and fetch data from them
-    //accounts "<YOUR ID>" {
+    //accounts "<YOUR ACCOUNT ID>" {
 	// Optional. Role ARN we want to assume when accessing this account
 	// role_arn = <YOUR_ROLE_ARN>
+	// Optional. Account ID we want to assume when accessing this account - override the block label
+	// account_id = <YOUR ACCOUNT ID>
 	// }
 	// Optional. by default assumes all regions
 	// regions = ["us-east-1", "us-west-2"]

client/testdata/assume_role.hcl

@@ -0,0 +1,28 @@
+cloudquery {
+  plugin_directory = "./cq/providers"
+  policy_directory = "./cq/policies"
+
+  provider "aws" {
+    source = "cloudquery/cq-provider-aws"
+    version = "latest"
+  }
+
+  connection {
+    dsn = "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable"
+  }
+}
+
+provider "aws" {
+  configuration {
+
+    accounts "default" {
+      role_arn = "${ASSUME_ROLE_ARN}"
+      account_id = "${ASSUME_ROLE_ACCOUNT_ID}"
+    }
+
+    max_retries = 20
+    max_backoff = 60
+  }
+  resources = [
+    "*"]
+}