From a4742b7f17684a825713b0b3494d46a9e424cd42 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 16 Jun 2022 22:18:57 +0100 Subject: [PATCH] feat: YAML support --- client/config.go | 84 ++++++++++++++++++++++++---------- client/testing.go | 5 +- go.mod | 4 +- go.sum | 7 +-- resources/provider/provider.go | 5 +- 5 files changed, 73 insertions(+), 32 deletions(-) diff --git a/client/config.go b/client/config.go index 50a96cd07..9cb4aabcb 100644 --- a/client/config.go +++ b/client/config.go @@ -1,39 +1,51 @@ package client +import "github.com/cloudquery/cq-provider-sdk/cqproto" + type Account struct { - ID string `hcl:",label"` + ID string `yaml:"id" hcl:",label"` AccountID string - AccountName string `hcl:"account_name,optional"` - LocalProfile string `hcl:"local_profile,optional"` - RoleARN string `hcl:"role_arn,optional"` - RoleSessionName string `hcl:"role_session_name,optional"` - ExternalID string `hcl:"external_id,optional"` - Regions []string `hcl:"regions,optional"` + AccountName string `yaml:"account_name,omitempty" hcl:"account_name,optional"` + LocalProfile string `yaml:"local_profile,omitempty" hcl:"local_profile,optional"` + RoleARN string `yaml:"role_arn,omitempty" hcl:"role_arn,optional"` + RoleSessionName string `yaml:"role_session_name,omitempty" hcl:"role_session_name,optional"` + ExternalID string `yaml:"external_id,omitempty" hcl:"external_id,optional"` + Regions []string `yaml:"regions,omitempty" hcl:"regions,optional"` source string } type AwsOrg struct { - OrganizationUnits []string `hcl:"organization_units,optional"` - AdminAccount *Account `hcl:"admin_account,block"` - MemberCredentials *Account `hcl:"member_trusted_principal,block"` - ChildAccountRoleName string `hcl:"member_role_name,optional"` - ChildAccountRoleSessionName string `hcl:"member_role_session_name,optional"` - ChildAccountExternalID string `hcl:"member_external_id,optional"` - ChildAccountRegions []string `hcl:"member_regions,optional"` + OrganizationUnits []string `yaml:"organization_units,omitempty" hcl:"organization_units,optional"` + AdminAccount *Account `yaml:"admin_account" hcl:"admin_account,block"` + MemberCredentials *Account `yaml:"member_trusted_principal" hcl:"member_trusted_principal,block"` + ChildAccountRoleName string `yaml:"member_role_name,omitempty" hcl:"member_role_name,optional"` + ChildAccountRoleSessionName string `yaml:"member_role_session_name,omitempty" hcl:"member_role_session_name,optional"` + ChildAccountExternalID string `yaml:"member_external_id,omitempty" hcl:"member_external_id,optional"` + ChildAccountRegions []string `yaml:"member_regions,omitempty" hcl:"member_regions,optional"` } type Config struct { - Regions []string `hcl:"regions,optional"` - Accounts []Account `hcl:"accounts,block"` - Organization *AwsOrg `hcl:"org,block"` - AWSDebug bool `hcl:"aws_debug,optional"` - MaxRetries int `hcl:"max_retries,optional" default:"10"` - MaxBackoff int `hcl:"max_backoff,optional" default:"30"` - GlobalRegion string `hcl:"global_region,optional" default:"us-east-1"` + Regions []string `yaml:"regions,omitempty" hcl:"regions,optional"` + Accounts []Account `yaml:"accounts" hcl:"accounts,block"` + Organization *AwsOrg `yaml:"org" hcl:"org,block"` + AWSDebug bool `yaml:"aws_debug,omitempty" hcl:"aws_debug,optional"` + MaxRetries int `yaml:"max_retries,omitempty" hcl:"max_retries,optional" default:"10"` + MaxBackoff int `yaml:"max_backoff,omitempty" hcl:"max_backoff,optional" default:"30"` + GlobalRegion string `yaml:"global_region,omitempty" hcl:"global_region,optional" default:"us-east-1"` + + requestedFormat cqproto.ConfigFormat +} + +func NewConfig(f cqproto.ConfigFormat) *Config { + return &Config{ + requestedFormat: f, + } } -func (Config) Example() string { - return ` configuration { +func (c Config) Example() string { + switch c.requestedFormat { + case cqproto.ConfigHCL: + return ` configuration { // Optional, Repeated. Add an 'accounts' block for every account you want to assume-role into and fetch data from. // accounts "" { // Optional. Role ARN we want to assume when accessing this account @@ -51,4 +63,30 @@ func (Config) Example() string { // max_backoff = 30 } ` + default: + return ` + # Optional, Repeated. Add an accounts block for every account you want to assume-role into and fetch data from. + # accounts: + # - id: + # Optional. Role ARN we want to assume when accessing this account + # role_arn: < YOUR_ROLE_ARN > + # Optional. Named profile in config or credential file from where CQ should grab credentials + # local_profile = < PROFILE_NAME > + + # Optional. by default assumes all regions + #regions: + # - us-east-1 + # us-west-2 + # Optional. Enable AWS SDK debug logging. + aws_debug: false + # The maximum number of times that a request will be retried for failures. Defaults to 10 retry attempts. + # max_retries: 10 + # The maximum back off delay between attempts. The backoff delays exponentially with a jitter based on the number of attempts. Defaults to 30 seconds. + // max_backoff: 30 +` + } +} + +func (c Config) Format() cqproto.ConfigFormat { + return c.requestedFormat } diff --git a/client/testing.go b/client/testing.go index 970bad1a5..6b767932c 100644 --- a/client/testing.go +++ b/client/testing.go @@ -3,6 +3,7 @@ package client import ( "testing" + "github.com/cloudquery/cq-provider-sdk/cqproto" "github.com/cloudquery/cq-provider-sdk/logging" "github.com/cloudquery/cq-provider-sdk/provider" "github.com/cloudquery/cq-provider-sdk/provider/diag" @@ -47,8 +48,8 @@ func AwsMockTestHelper(t *testing.T, table *schema.Table, builder func(*testing. ResourceMap: map[string]*schema.Table{ "test_resource": table, }, - Config: func() provider.Config { - return &Config{} + Config: func(f cqproto.ConfigFormat) provider.Config { + return NewConfig(f) }, }, Config: cfg, diff --git a/go.mod b/go.mod index 2174d6a24..167438149 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/basgys/goxml2json v1.1.0 github.com/bxcodec/faker v2.0.1+incompatible github.com/cloudquery/cq-gen v0.0.5 - github.com/cloudquery/cq-provider-sdk v0.11.3 + github.com/cloudquery/cq-provider-sdk v0.11.4-0.20220616202428-10405c8f1f24 github.com/cloudquery/faker/v3 v3.7.5 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8 github.com/golang/mock v1.6.0 @@ -168,5 +168,5 @@ require ( google.golang.org/grpc v1.42.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8aba3b047..d37bfa026 100644 --- a/go.sum +++ b/go.sum @@ -363,8 +363,8 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cloudquery/cq-gen v0.0.5 h1:yhDhM4RCqqGLZulDzfA51VMy0b7TIdtifoEiZXnFbUc= github.com/cloudquery/cq-gen v0.0.5/go.mod h1:zrjBcuCGtED9P4RzA4gK+P3loxn0Ij1wEcBZX97JTnI= github.com/cloudquery/cq-provider-sdk v0.8.2/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= -github.com/cloudquery/cq-provider-sdk v0.11.3 h1:o/qorzRlEB9zFOugmGxCvnW9E6L829vqDpw7Coe/dbY= -github.com/cloudquery/cq-provider-sdk v0.11.3/go.mod h1:6AepQERMVIk8igb8HxZK6ThI2KaftOkwm5LsN1cSWKs= +github.com/cloudquery/cq-provider-sdk v0.11.4-0.20220616202428-10405c8f1f24 h1:hV9iu1EGpnfnuib7U4i829iZAkV8IM91y5ZEXFN/wQg= +github.com/cloudquery/cq-provider-sdk v0.11.4-0.20220616202428-10405c8f1f24/go.mod h1:IWV+RHvDZ6ugRLp3V5R78WMOUahxxEF1SOP6lePN+QI= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cloudquery/faker/v3 v3.7.5 h1:G7ANdEEcm8TvAAjIwNWSLrYK36CFCiSlrCqOTGCccL0= github.com/cloudquery/faker/v3 v3.7.5/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= @@ -1814,8 +1814,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/resources/provider/provider.go b/resources/provider/provider.go index 981bc31b1..c2a70fa60 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -59,6 +59,7 @@ import ( "github.com/cloudquery/cq-provider-aws/resources/services/wafv2" "github.com/cloudquery/cq-provider-aws/resources/services/workspaces" "github.com/cloudquery/cq-provider-aws/resources/services/xray" + "github.com/cloudquery/cq-provider-sdk/cqproto" "github.com/cloudquery/cq-provider-sdk/provider" "github.com/cloudquery/cq-provider-sdk/provider/module" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -244,8 +245,8 @@ func Provider() *provider.Provider { "xray.sampling_rules": xray.SamplingRules(), //"iot.security_profiles": iot.IotSecurityProfiles(), //TODO disabled because of api error NotFoundException: No method found matching route security-profiles for http method GET. }, - Config: func() provider.Config { - return &client.Config{} + Config: func(f cqproto.ConfigFormat) provider.Config { + return client.NewConfig(f) }, } }