From 2f80c9db68d03bf6f17baface5a4bb21e157f7b4 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 10:25:22 +0000 Subject: [PATCH 01/15] Add table and plugin level defaults for ShouldIgnoreError and RetryConfig. Closes #257 --- plugin/hydrate_call.go | 39 ++++ plugin/plugin.go | 15 +- plugin/required_hydrate_calls.go | 3 +- plugin/table.go | 38 +++- plugin/table_test.go | 298 +++++++++++++++++++++++++++++-- plugin/table_validate.go | 41 +++-- 6 files changed, 394 insertions(+), 40 deletions(-) diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 34da3fdf..483698d5 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -2,6 +2,8 @@ package plugin import ( "context" + "fmt" + "strings" "github.com/turbot/go-kit/helpers" ) @@ -33,10 +35,47 @@ type HydrateConfig struct { Depends []HydrateFunc } +func (c *HydrateConfig) DefaultTo(defaultConfig *HydrateConfig) { + if defaultConfig == nil { + return + } + if c.RetryConfig == nil { + c.RetryConfig = defaultConfig.RetryConfig + } + if c.Depends == nil { + c.Depends = defaultConfig.Depends + } +} + +func (c *HydrateConfig) String() interface{} { + shouldIgnoreErrorString := "" + if c.ShouldIgnoreError != nil { + shouldIgnoreErrorString = helpers.GetFunctionName(c.Func) + } + var dependsStrings = make([]string, len(c.Depends)) + for i, dep := range c.Depends { + dependsStrings[i] = helpers.GetFunctionName(dep) + } + return fmt.Sprintf(`Func: %s +MaxConcurrency: %d +RetryConfig: %s +ShouldIgnoreError: %s +Depends: %s`, + helpers.GetFunctionName(c.Func), + c.MaxConcurrency, + c.RetryConfig.String(), + shouldIgnoreErrorString, + strings.Join(dependsStrings, ",")) +} + type RetryConfig struct { ShouldRetryError ErrorPredicate } +func (c RetryConfig) String() interface{} { + return fmt.Sprintf("ShouldRetryError: %s", helpers.GetFunctionName(c.ShouldRetryError)) +} + // DefaultConcurrencyConfig contains plugin level config to define default hydrate concurrency // - this is used if no HydrateConfig is specified for a specific call type DefaultConcurrencyConfig struct { diff --git a/plugin/plugin.go b/plugin/plugin.go index a3181154..94bde002 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -30,18 +30,21 @@ var validSchemaModes = []string{SchemaModeStatic, SchemaModeDynamic} // Plugin is an object used to build all necessary data for a given query type Plugin struct { - Name string - Logger hclog.Logger + Name string + Logger hclog.Logger + // TableMap is a map of all the tables in the plugin, keyed by the table name TableMap map[string]*Table // TableMapFunc is a callback function which can be used to populate the table map // this con optionally be provided by the plugin, and allows the connection config to be used in the table creation // (connection config is not available at plugin creation time) TableMapFunc func(ctx context.Context, p *Plugin) (map[string]*Table, error) - DefaultTransform *transform.ColumnTransforms - DefaultGetConfig *GetConfig - DefaultConcurrency *DefaultConcurrencyConfig - DefaultRetryConfig *RetryConfig + // + DefaultTransform *transform.ColumnTransforms + DefaultGetConfig *GetConfig + DefaultConcurrency *DefaultConcurrencyConfig + DefaultRetryConfig *RetryConfig + DefaultShouldIgnoreError ErrorPredicate // every table must implement these columns RequiredColumns []*Column ConnectionConfigSchema *ConnectionConfigSchema diff --git a/plugin/required_hydrate_calls.go b/plugin/required_hydrate_calls.go index aae38198..d7f08b88 100644 --- a/plugin/required_hydrate_calls.go +++ b/plugin/required_hydrate_calls.go @@ -24,7 +24,8 @@ func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc) { // if the resolved hydrate call is NOT the same as the fetch call, add to the map of hydrate functions to call if hydrateName != c.fetchCallName { - if _, ok := c.requiredHydrateCalls[hydrateName]; !ok { + if _, ok := c.requiredHydrateCalls[hydrateName]; ok { + return } // get the config for this hydrate function diff --git a/plugin/table.go b/plugin/table.go index 2602a472..02255818 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -16,14 +16,18 @@ type Table struct { // table description Description string // column definitions - Columns []*Column - List *ListConfig - Get *GetConfig - GetMatrixItem MatrixItemFunc - DefaultTransform *transform.ColumnTransforms + Columns []*Column + List *ListConfig + Get *GetConfig + GetMatrixItem MatrixItemFunc + DefaultTransform *transform.ColumnTransforms + DefaultShouldIgnoreError ErrorPredicate + DefaultRetryConfig *RetryConfig + // the parent plugin object Plugin *Plugin // definitions of dependencies between hydrate functions + // Deprecated: used HydrateConfig HydrateDependencies []HydrateDependencies HydrateConfig []HydrateConfig // map of hydrate function name to columns it provides @@ -97,10 +101,16 @@ func (t *Table) getFetchFunc(fetchType fetchType) HydrateFunc { return t.List.Hydrate } return t.Get.Hydrate - } +// search through HydrateConfig and HydrateDependencies finding a function with the given name +// if found return its dependencies func (t *Table) getHydrateDependencies(hydrateFuncName string) []HydrateFunc { + for _, d := range t.HydrateConfig { + if helpers.GetFunctionName(d.Func) == hydrateFuncName { + return d.Depends + } + } for _, d := range t.HydrateDependencies { if helpers.GetFunctionName(d.Func) == hydrateFuncName { return d.Depends @@ -115,11 +125,25 @@ func (t *Table) getHydrateConfig(hydrateFuncName string) *HydrateConfig { for _, d := range t.HydrateConfig { if helpers.GetFunctionName(d.Func) == hydrateFuncName { config = &d + break } } + // now use default values if needed if config.RetryConfig == nil { - config.RetryConfig = t.Plugin.DefaultRetryConfig + if t.DefaultRetryConfig != nil { + config.RetryConfig = t.DefaultRetryConfig + } else { + config.RetryConfig = t.Plugin.DefaultRetryConfig + } + } + if config.ShouldIgnoreError == nil { + if t.DefaultShouldIgnoreError != nil { + config.ShouldIgnoreError = t.DefaultShouldIgnoreError + } else { + config.ShouldIgnoreError = t.Plugin.DefaultShouldIgnoreError + } } + // if no hydrate dependencies are specified in the hydrate config, check the deprecated "HydrateDependencies" property if config.Depends == nil { config.Depends = t.getHydrateDependencies(hydrateFuncName) diff --git a/plugin/table_test.go b/plugin/table_test.go index 977d9bff..85de7e2f 100644 --- a/plugin/table_test.go +++ b/plugin/table_test.go @@ -3,13 +3,15 @@ package plugin import ( "context" "fmt" + "sort" "strings" "testing" "github.com/turbot/steampipe-plugin-sdk/v2/grpc/proto" ) -//// isGet //// +/////////////////////////////// +//// isGet type isGetTest struct { table Table @@ -169,7 +171,8 @@ func TestIsGet(t *testing.T) { //} } -//// requiredHydrateCallBuilder //// +/////////////////////////////// +//// requiredHydrateCallBuilder // declared in plugin_test // listHydrate, getHydrate, hydrate1, hydrate2, hydrate3, hydrate4 @@ -216,9 +219,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ }, columns: []string{"c1"}, fetchType: fetchTypeList, - expected: []*HydrateCall{{Func: hydrate1}}, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1"}}, }, - "list - 1 hydrate, depends": { + "list - 1 hydrate, depends [HydrateDependencies]": { table: Table{ Name: "table", Columns: []*Column{ @@ -227,14 +230,14 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ }, List: &ListConfig{Hydrate: listHydrate}, Get: &GetConfig{Hydrate: getHydrate}, - HydrateDependencies: []HydrateDependencies{{hydrate1, []HydrateFunc{hydrate2}}}, + HydrateDependencies: []HydrateDependencies{{Func: hydrate1, Depends: []HydrateFunc{hydrate2}}}, Plugin: &Plugin{}, }, columns: []string{"c1"}, fetchType: fetchTypeList, - expected: []*HydrateCall{{Func: hydrate1, Depends: []string{"hydrate2"}}, {Func: hydrate2}}, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1", Depends: []string{"hydrate2"}}, {Func: hydrate2, Name: "hydrate2"}}, }, - "get - 2 hydrate, depends": { + "get - 2 hydrate, depends [HydrateDependencies]": { table: Table{ Name: "table", Columns: []*Column{ @@ -249,9 +252,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ }, columns: []string{"c1", "c2"}, fetchType: fetchTypeGet, - expected: []*HydrateCall{{Func: hydrate1, Depends: []string{"hydrate3"}}, {Func: hydrate3}, {Func: hydrate2}}, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1", Depends: []string{"hydrate3"}}, {Func: hydrate3, Name: "hydrate3"}, {Func: hydrate2, Name: "hydrate2"}}, }, - "get - 2 depends": { + "get - 2 depends [HydrateDependencies]": { table: Table{ Name: "table", Columns: []*Column{ @@ -269,9 +272,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ }, columns: []string{"c1"}, fetchType: fetchTypeGet, - expected: []*HydrateCall{{Func: hydrate1, Depends: []string{"hydrate2"}}, {Func: hydrate2, Depends: []string{"hydrate3"}}, {Func: hydrate3}}, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1", Depends: []string{"hydrate2"}}, {Func: hydrate2, Name: "hydrate2", Depends: []string{"hydrate3"}}, {Func: hydrate3, Name: "hydrate3"}}, }, - "get - unreferenced depends": { + "get - unreferenced depends [HydrateDependencies]": { table: Table{ Name: "table", Columns: []*Column{ @@ -289,7 +292,81 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ }, columns: []string{"c3"}, fetchType: fetchTypeGet, - expected: []*HydrateCall{{Func: hydrate3}}, + expected: []*HydrateCall{{Func: hydrate3, Name: "hydrate3"}}, + }, + + "list - 1 hydrate, depends": { + table: Table{ + Name: "table", + Columns: []*Column{ + {Name: "c1", Hydrate: hydrate1}, + {Name: "c2"}, + }, + List: &ListConfig{Hydrate: listHydrate}, + Get: &GetConfig{Hydrate: getHydrate}, + HydrateConfig: []HydrateConfig{{Func: hydrate1, Depends: []HydrateFunc{hydrate2}}}, + Plugin: &Plugin{}, + }, + columns: []string{"c1"}, + fetchType: fetchTypeList, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1", Depends: []string{"hydrate2"}}, {Func: hydrate2, Name: "hydrate2"}}, + }, + "get - 2 hydrate, depends": { + table: Table{ + Name: "table", + Columns: []*Column{ + {Name: "c1", Hydrate: hydrate1}, + {Name: "c2", Hydrate: hydrate2}, + {Name: "c3", Hydrate: hydrate3}, + }, + List: &ListConfig{Hydrate: listHydrate}, + Get: &GetConfig{Hydrate: getHydrate}, + HydrateConfig: []HydrateConfig{{Func: hydrate1, Depends: []HydrateFunc{hydrate3}}}, + Plugin: &Plugin{}, + }, + columns: []string{"c1", "c2"}, + fetchType: fetchTypeGet, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1", Depends: []string{"hydrate3"}}, {Func: hydrate3, Name: "hydrate3"}, {Func: hydrate2, Name: "hydrate2"}}, + }, + "get - 2 depends": { + table: Table{ + Name: "table", + Columns: []*Column{ + {Name: "c1", Hydrate: hydrate1}, + {Name: "c2", Hydrate: hydrate2}, + {Name: "c3", Hydrate: hydrate3}, + }, + List: &ListConfig{Hydrate: listHydrate}, + Get: &GetConfig{Hydrate: getHydrate}, + HydrateConfig: []HydrateConfig{ + {Func: hydrate1, Depends: []HydrateFunc{hydrate2}}, + {Func: hydrate2, Depends: []HydrateFunc{hydrate3}}, + }, + Plugin: &Plugin{}, + }, + columns: []string{"c1"}, + fetchType: fetchTypeGet, + expected: []*HydrateCall{{Func: hydrate1, Name: "hydrate1", Depends: []string{"hydrate2"}}, {Func: hydrate2, Name: "hydrate2", Depends: []string{"hydrate3"}}, {Func: hydrate3, Name: "hydrate3"}}, + }, + "get - unreferenced depends": { + table: Table{ + Name: "table", + Columns: []*Column{ + {Name: "c1", Hydrate: hydrate1}, + {Name: "c2", Hydrate: hydrate2}, + {Name: "c3", Hydrate: hydrate3}, + }, + List: &ListConfig{Hydrate: listHydrate}, + Get: &GetConfig{Hydrate: getHydrate}, + HydrateConfig: []HydrateConfig{ + {Func: hydrate1, Depends: []HydrateFunc{hydrate2}}, + {Func: hydrate2, Depends: []HydrateFunc{hydrate3}}, + }, + Plugin: &Plugin{}, + }, + columns: []string{"c3"}, + fetchType: fetchTypeGet, + expected: []*HydrateCall{{Func: hydrate3, Name: "hydrate3"}}, }, } @@ -313,6 +390,7 @@ func hydrateArrayToString(calls []*HydrateCall) string { for _, c := range calls { strs = append(strs, hydrateCallToString(c)) } + sort.Strings(strs) return strings.Join(strs, "\n") } @@ -327,3 +405,199 @@ func hydrateCallToString(call *HydrateCall) string { } return str } + +/////////////////////////////// +//// getHydrateConfig + +type getHydrateConfigTest struct { + table *Table + funcName string + expected *HydrateConfig +} + +var getHydrateConfigTestTableWithDefaults = &Table{ + Name: "test", + Plugin: getHydrateConfigTestPlugin, + DefaultShouldIgnoreError: shouldIgnoreErrorTableDefault, + DefaultRetryConfig: &RetryConfig{ + ShouldRetryError: shouldRetryErrorTableDefault, + }, + HydrateConfig: []HydrateConfig{ + { + Func: hydrate1, + MaxConcurrency: 1, + Depends: []HydrateFunc{hydrate2, hydrate3}, + }, + { + Func: hydrate2, + MaxConcurrency: 2, + ShouldIgnoreError: shouldIgnoreError1, + }, + { + Func: hydrate3, + MaxConcurrency: 3, + RetryConfig: &RetryConfig{shouldRetryError1}, + }, + { + Func: hydrate4, + MaxConcurrency: 4, + ShouldIgnoreError: shouldIgnoreError2, + RetryConfig: &RetryConfig{shouldRetryError2}, + }, + }, +} + +var getHydrateConfigTestTableNoDefaults = &Table{ + Name: "test", + Plugin: getHydrateConfigTestPlugin, + HydrateConfig: []HydrateConfig{ + { + Func: hydrate1, + MaxConcurrency: 1, + Depends: []HydrateFunc{hydrate2, hydrate3}, + }, + { + Func: hydrate2, + MaxConcurrency: 2, + ShouldIgnoreError: shouldIgnoreError1, + }, + { + Func: hydrate3, + MaxConcurrency: 3, + RetryConfig: &RetryConfig{shouldRetryError1}, + }, + }, +} + +var getHydrateConfigTestPlugin = &Plugin{ + Name: "test", + DefaultShouldIgnoreError: shouldIgnoreErrorPluginDefault, + DefaultRetryConfig: &RetryConfig{ + ShouldRetryError: shouldRetryErrorPluginDefault, + }, +} + +var testCasesGetHydrateConfig = map[string]getHydrateConfigTest{ + "tables default retry and should ignore": { + table: getHydrateConfigTestTableWithDefaults, + funcName: "hydrate1", + + expected: &HydrateConfig{ + Func: hydrate1, + MaxConcurrency: 1, + RetryConfig: &RetryConfig{shouldRetryErrorTableDefault}, + ShouldIgnoreError: shouldIgnoreErrorTableDefault, + Depends: []HydrateFunc{hydrate2, hydrate3}, + }, + }, + "table default retry": { + table: getHydrateConfigTestTableWithDefaults, + funcName: "hydrate2", + + expected: &HydrateConfig{ + Func: hydrate2, + MaxConcurrency: 2, + RetryConfig: &RetryConfig{shouldRetryErrorTableDefault}, + ShouldIgnoreError: shouldIgnoreError1, + }, + }, + "tables default should ignore": { + table: getHydrateConfigTestTableWithDefaults, + funcName: "hydrate3", + + expected: &HydrateConfig{ + Func: hydrate3, + MaxConcurrency: 3, + RetryConfig: &RetryConfig{shouldRetryError1}, + ShouldIgnoreError: shouldIgnoreErrorTableDefault, + }, + }, + "plugin default retry and should ignore": { + table: getHydrateConfigTestTableNoDefaults, + funcName: "hydrate1", + + expected: &HydrateConfig{ + Func: hydrate1, + MaxConcurrency: 1, + RetryConfig: &RetryConfig{shouldRetryErrorPluginDefault}, + ShouldIgnoreError: shouldIgnoreErrorPluginDefault, + Depends: []HydrateFunc{hydrate2, hydrate3}, + }, + }, + "plugin default retry": { + table: getHydrateConfigTestTableNoDefaults, + funcName: "hydrate2", + + expected: &HydrateConfig{ + Func: hydrate2, + MaxConcurrency: 2, + RetryConfig: &RetryConfig{shouldRetryErrorPluginDefault}, + ShouldIgnoreError: shouldIgnoreError1, + }, + }, + "plugin default should ignore": { + table: getHydrateConfigTestTableNoDefaults, + funcName: "hydrate3", + + expected: &HydrateConfig{ + Func: hydrate3, + MaxConcurrency: 3, + RetryConfig: &RetryConfig{shouldRetryError1}, + ShouldIgnoreError: shouldIgnoreErrorPluginDefault, + }, + }, + "no defaults": { + table: getHydrateConfigTestTableWithDefaults, + funcName: "hydrate4", + + expected: &HydrateConfig{ + Func: hydrate4, + MaxConcurrency: 4, + RetryConfig: &RetryConfig{shouldRetryError2}, + ShouldIgnoreError: shouldRetryError2, + }, + }, +} + +func shouldIgnoreErrorTableDefault(err error) bool { + return true +} +func shouldIgnoreErrorPluginDefault(err error) bool { + return true +} +func shouldIgnoreError1(err error) bool { + return true +} +func shouldIgnoreError2(err error) bool { + return true +} +func shouldIgnoreError3(err error) bool { + return true +} +func shouldIgnoreError4(err error) bool { + return true +} + +func shouldRetryErrorTableDefault(err error) bool { + return true +} +func shouldRetryErrorPluginDefault(err error) bool { + return true +} +func shouldRetryError1(err error) bool { + return true +} +func shouldRetryError2(err error) bool { + return true +} + +func TestGetHydrateConfig(t *testing.T) { + for name, test := range testCasesGetHydrateConfig { + result := test.table.getHydrateConfig(test.funcName) + actualString := result.String() + expectedString := test.expected.String() + if expectedString != actualString { + t.Errorf("Test: '%s'' FAILED : expected: \n\n%v\n\ngot: \n\n%v", name, expectedString, actualString) + } + } +} diff --git a/plugin/table_validate.go b/plugin/table_validate.go index 9332dfcd..58cc8596 100644 --- a/plugin/table_validate.go +++ b/plugin/table_validate.go @@ -23,7 +23,7 @@ func (t *Table) validate(name string, requiredColumns []*Column) []string { validationErrors = append(validationErrors, t.validateListAndGetConfig()...) // verify hydrate dependencies are valid - // the map entries are strings - ensure they correpond to actual functions + // the map entries are strings - ensure they correspond to actual functions validationErrors = append(validationErrors, t.validateHydrateDependencies()...) // verify any ALL key columns do not duplicate columns between ALL fields @@ -106,8 +106,13 @@ func (t *Table) validateListAndGetConfig() []string { } func (t *Table) validateHydrateDependencies() []string { + // only 1 of HydrateDependencies and HydrateConfig) may be set + if len(t.HydrateDependencies) != 0 && len(t.HydrateConfig) != 0 { + return []string{fmt.Sprintf("table '%s' defines both HydrateDependencies and HydrateConfig", t.Name)} + } + var validationErrors []string - if t.HydrateDependencies != nil { + if len(t.HydrateDependencies)+len(t.HydrateConfig) != 0 { if t.List != nil { deps := t.getHydrateDependencies(helpers.GetFunctionName(t.List.Hydrate)) if len(deps) > 0 { @@ -128,24 +133,32 @@ func (t *Table) validateHydrateDependencies() []string { } func (t *Table) detectCyclicHydrateDependencies() string { - var dependencies = topsort.NewGraph() - dependencies.AddNode("root") - for _, hydrateDeps := range t.HydrateDependencies { - name := helpers.GetFunctionName(hydrateDeps.Func) - if !dependencies.ContainsNode(name) { - dependencies.AddNode(name) + var dependencyGraph = topsort.NewGraph() + dependencyGraph.AddNode("root") + + updateDependencyGraph := func(hydrateFunc HydrateFunc, hydrateDepends []HydrateFunc) { + name := helpers.GetFunctionName(hydrateFunc) + if !dependencyGraph.ContainsNode(name) { + dependencyGraph.AddNode(name) } - dependencies.AddEdge("root", name) - for _, dep := range hydrateDeps.Depends { + dependencyGraph.AddEdge("root", name) + for _, dep := range hydrateDepends { depName := helpers.GetFunctionName(dep) - if !dependencies.ContainsNode(depName) { - dependencies.AddNode(depName) + if !dependencyGraph.ContainsNode(depName) { + dependencyGraph.AddNode(depName) } - dependencies.AddEdge(name, depName) + dependencyGraph.AddEdge(name, depName) } } - if _, err := dependencies.TopSort("root"); err != nil { + for _, hydrateDeps := range t.HydrateDependencies { + updateDependencyGraph(hydrateDeps.Func, hydrateDeps.Depends) + } + for _, hydrateConfig := range t.HydrateConfig { + updateDependencyGraph(hydrateConfig.Func, hydrateConfig.Depends) + } + + if _, err := dependencyGraph.TopSort("root"); err != nil { return strings.Replace(err.Error(), "Cycle error", "Hydration dependencies contains cycle: ", 1) } From aaeb7a02c2cbe54ceec73ef255dd4952e2ed9f2a Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 10:47:02 +0000 Subject: [PATCH 02/15] tidy --- plugin/concurrency.go | 8 ++++ plugin/hydrate_call.go | 78 ---------------------------------- plugin/hydrate_config.go | 50 ++++++++++++++++++++++ plugin/hydrate_data.go | 9 ++++ plugin/hydrate_dependencies.go | 8 ++++ plugin/hydrate_func.go | 8 ++++ plugin/retry_config.go | 15 +++++++ 7 files changed, 98 insertions(+), 78 deletions(-) create mode 100644 plugin/hydrate_config.go create mode 100644 plugin/hydrate_data.go create mode 100644 plugin/hydrate_dependencies.go create mode 100644 plugin/hydrate_func.go create mode 100644 plugin/retry_config.go diff --git a/plugin/concurrency.go b/plugin/concurrency.go index a71bd242..8e5b4088 100644 --- a/plugin/concurrency.go +++ b/plugin/concurrency.go @@ -5,6 +5,14 @@ import ( "sync" ) +// DefaultConcurrencyConfig contains plugin level config to define default hydrate concurrency +// - this is used if no HydrateConfig is specified for a specific call +type DefaultConcurrencyConfig struct { + // max number of ALL hydrate calls in progress + TotalMaxConcurrency int + DefaultMaxConcurrency int +} + // ConcurrencyManager struct ensures that hydrate functions stay within concurrency limits type ConcurrencyManager struct { mut sync.Mutex diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 483698d5..208581eb 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -2,88 +2,10 @@ package plugin import ( "context" - "fmt" - "strings" "github.com/turbot/go-kit/helpers" ) -// HydrateData contains the input data passed to every hydrate function -type HydrateData struct { - // if there was a parent-child list call, store the parent list item - ParentItem interface{} - Item interface{} - HydrateResults map[string]interface{} -} - -// HydrateFunc is a function which retrieves some or all row data for a single row item. -type HydrateFunc func(context.Context, *QueryData, *HydrateData) (interface{}, error) - -// HydrateDependencies defines the hydrate function dependencies - other hydrate functions which must be run first -// Deprecated: used HydrateConfig -type HydrateDependencies struct { - Func HydrateFunc - Depends []HydrateFunc -} - -// HydrateConfig defines the hydrate function configurations, Name, Maximum number of concurrent calls to be allowed, dependencies -type HydrateConfig struct { - Func HydrateFunc - MaxConcurrency int - RetryConfig *RetryConfig - ShouldIgnoreError ErrorPredicate - Depends []HydrateFunc -} - -func (c *HydrateConfig) DefaultTo(defaultConfig *HydrateConfig) { - if defaultConfig == nil { - return - } - if c.RetryConfig == nil { - c.RetryConfig = defaultConfig.RetryConfig - } - if c.Depends == nil { - c.Depends = defaultConfig.Depends - } -} - -func (c *HydrateConfig) String() interface{} { - shouldIgnoreErrorString := "" - if c.ShouldIgnoreError != nil { - shouldIgnoreErrorString = helpers.GetFunctionName(c.Func) - } - var dependsStrings = make([]string, len(c.Depends)) - for i, dep := range c.Depends { - dependsStrings[i] = helpers.GetFunctionName(dep) - } - return fmt.Sprintf(`Func: %s -MaxConcurrency: %d -RetryConfig: %s -ShouldIgnoreError: %s -Depends: %s`, - helpers.GetFunctionName(c.Func), - c.MaxConcurrency, - c.RetryConfig.String(), - shouldIgnoreErrorString, - strings.Join(dependsStrings, ",")) -} - -type RetryConfig struct { - ShouldRetryError ErrorPredicate -} - -func (c RetryConfig) String() interface{} { - return fmt.Sprintf("ShouldRetryError: %s", helpers.GetFunctionName(c.ShouldRetryError)) -} - -// DefaultConcurrencyConfig contains plugin level config to define default hydrate concurrency -// - this is used if no HydrateConfig is specified for a specific call -type DefaultConcurrencyConfig struct { - // max number of ALL hydrate calls in progress - TotalMaxConcurrency int - DefaultMaxConcurrency int -} - // HydrateCall struct encapsulates a hydrate call, its config and dependencies type HydrateCall struct { Func HydrateFunc diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go new file mode 100644 index 00000000..eeefeb7c --- /dev/null +++ b/plugin/hydrate_config.go @@ -0,0 +1,50 @@ +package plugin + +import ( + "fmt" + "strings" + + "github.com/turbot/go-kit/helpers" +) + +// HydrateConfig defines the hydrate function configurations, Name, Maximum number of concurrent calls to be allowed, dependencies +type HydrateConfig struct { + Func HydrateFunc + MaxConcurrency int + RetryConfig *RetryConfig + ShouldIgnoreError ErrorPredicate + Depends []HydrateFunc +} + +func (c *HydrateConfig) DefaultTo(defaultConfig *HydrateConfig) { + if defaultConfig == nil { + return + } + if c.RetryConfig == nil { + c.RetryConfig = defaultConfig.RetryConfig + } + if c.Depends == nil { + c.Depends = defaultConfig.Depends + } +} + +func (c *HydrateConfig) String() interface{} { + shouldIgnoreErrorString := "" + if c.ShouldIgnoreError != nil { + shouldIgnoreErrorString = helpers.GetFunctionName(c.Func) + } + var dependsStrings = make([]string, len(c.Depends)) + for i, dep := range c.Depends { + dependsStrings[i] = helpers.GetFunctionName(dep) + } + return fmt.Sprintf(`Func: %s +MaxConcurrency: %d +RetryConfig: %s +ShouldIgnoreError: %s +Depends: %s`, + helpers.GetFunctionName(c.Func), + c.MaxConcurrency, + c.RetryConfig.String(), + shouldIgnoreErrorString, + strings.Join(dependsStrings, ",")) +} diff --git a/plugin/hydrate_data.go b/plugin/hydrate_data.go new file mode 100644 index 00000000..4f2e1bb5 --- /dev/null +++ b/plugin/hydrate_data.go @@ -0,0 +1,9 @@ +package plugin + +// HydrateData contains the input data passed to every hydrate function +type HydrateData struct { + // if there was a parent-child list call, store the parent list item + ParentItem interface{} + Item interface{} + HydrateResults map[string]interface{} +} diff --git a/plugin/hydrate_dependencies.go b/plugin/hydrate_dependencies.go new file mode 100644 index 00000000..6b498e04 --- /dev/null +++ b/plugin/hydrate_dependencies.go @@ -0,0 +1,8 @@ +package plugin + +// HydrateDependencies defines the hydrate function dependencies - other hydrate functions which must be run first +// Deprecated: used HydrateConfig +type HydrateDependencies struct { + Func HydrateFunc + Depends []HydrateFunc +} diff --git a/plugin/hydrate_func.go b/plugin/hydrate_func.go new file mode 100644 index 00000000..d9d429f8 --- /dev/null +++ b/plugin/hydrate_func.go @@ -0,0 +1,8 @@ +package plugin + +import ( + "context" +) + +// HydrateFunc is a function which retrieves some or all row data for a single row item. +type HydrateFunc func(context.Context, *QueryData, *HydrateData) (interface{}, error) diff --git a/plugin/retry_config.go b/plugin/retry_config.go new file mode 100644 index 00000000..abc40b4a --- /dev/null +++ b/plugin/retry_config.go @@ -0,0 +1,15 @@ +package plugin + +import ( + "fmt" + + "github.com/turbot/go-kit/helpers" +) + +type RetryConfig struct { + ShouldRetryError ErrorPredicate +} + +func (c RetryConfig) String() interface{} { + return fmt.Sprintf("ShouldRetryError: %s", helpers.GetFunctionName(c.ShouldRetryError)) +} From 408b833453208c0322f6ad799bd5a5f3134fe38b Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 11:59:12 +0000 Subject: [PATCH 03/15] fix unit tests, add unit tests to acc workflow --- .github/workflows/acceptance-test.yml | 5 +++++ cache/index_item.go | 2 +- cache/query_cache.go | 2 +- grpc/proto/qual.go | 9 ++++++-- plugin/connection_config_test.go | 18 ++++++++++------ plugin/hydrate_config.go | 7 ++++--- plugin/plugin_test.go | 5 +++-- plugin/table_test.go | 30 ++++++++++----------------- plugin/table_validate.go | 8 +++---- 9 files changed, 48 insertions(+), 38 deletions(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index d8a1e8a3..4fe4b774 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -15,6 +15,11 @@ jobs: with: go-version: 1.16 + - name: Run CLI Unit Tests + run: | + go clean -testcache + go test -timeout 30s ./... + - name: Checkout Steampipe uses: actions/checkout@v2 with: diff --git a/cache/index_item.go b/cache/index_item.go index 80a676eb..a1d172b8 100644 --- a/cache/index_item.go +++ b/cache/index_item.go @@ -86,7 +86,7 @@ func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals, keyColum // isSubset means all data returned by check quals is returned by index quals isSubset = checkQuals.IsASubsetOf(indexQuals) } else { - log.Printf("[TRACE] SatisfiesQuals index item has qual for %s which check quals do not - NOT SATISFIED") + log.Printf("[TRACE] SatisfiesQuals index item has qual for %s which check quals do not - NOT SATISFIED", col) } log.Printf("[TRACE] get check qual %v, isSubset %v", ok, isSubset) if !ok || !isSubset { diff --git a/cache/query_cache.go b/cache/query_cache.go index 5233888e..72338aef 100644 --- a/cache/query_cache.go +++ b/cache/query_cache.go @@ -307,7 +307,7 @@ func (c *QueryCache) logMetrics() { log.Printf("[TRACE] ------------------------------------ ") log.Printf("[TRACE] Cache Metrics ") log.Printf("[TRACE] ------------------------------------ ") - log.Printf("[TRACE] MaxCost: %d", c.cache.MaxCost) + log.Printf("[TRACE] MaxCost: %d", c.cache.MaxCost()) log.Printf("[TRACE] KeysAdded: %d", c.cache.Metrics.KeysAdded()) log.Printf("[TRACE] CostAdded: %d", c.cache.Metrics.CostAdded()) log.Printf("[TRACE] KeysEvicted: %d", c.cache.Metrics.KeysEvicted()) diff --git a/grpc/proto/qual.go b/grpc/proto/qual.go index c15902c0..87e04ac8 100644 --- a/grpc/proto/qual.go +++ b/grpc/proto/qual.go @@ -3,12 +3,12 @@ package proto import "log" func (x *Qual) Equals(other *Qual) bool { - log.Printf("[TRACE] me %s, other %s", x.String(), other.String()) + //log.Printf("[TRACE] me %s, other %s", x.String(), other.String()) return x.String() == other.String() } func (x *Qual) IsASubsetOf(other *Qual) bool { - log.Printf("[TRACE] IsASubsetOf me %+v, other %+v", x, other) + //log.Printf("[TRACE] IsASubsetOf me %+v, other %+v", x, other) operator, ok := x.Operator.(*Qual_StringValue) if !ok { log.Printf("[TRACE] IsASubsetOf my operator is not a string - returning false") @@ -23,6 +23,11 @@ func (x *Qual) IsASubsetOf(other *Qual) bool { log.Printf("[TRACE] IsASubsetOf Value nil - returning false") return false } + // check the fields are the same + if x.FieldName != other.FieldName { + log.Printf("[TRACE] IsASubsetOf field names different - returning false") + return false + } switch value := x.Value.Value.(type) { case *QualValue_StringValue: diff --git a/plugin/connection_config_test.go b/plugin/connection_config_test.go index cd7c8e99..05d00ea8 100644 --- a/plugin/connection_config_test.go +++ b/plugin/connection_config_test.go @@ -106,6 +106,9 @@ var testCasesParseConfig = map[string]parseConfigTest{ Type: schema.TypeString, Required: true, }, + "count": { + Type: schema.TypeInt, + }, }, }, expectedFunc: func(res interface{}) bool { @@ -438,12 +441,15 @@ func TestParseConnectionConfig(t *testing.T) { } continue } - if test.expectedFunc != nil && !test.expectedFunc(config) { - t.Errorf(`Test: '%s' FAILED : expect verification func failed`, name) - } - if !reflect.DeepEqual(config, test.expected) { - fmt.Printf("") - t.Errorf(`Test: '%s' FAILED : expected %v, got %v`, name, test.expected, config) + if test.expectedFunc != nil { + if !test.expectedFunc(config) { + t.Errorf(`Test: '%s' FAILED : expect verification func failed`, name) + } + } else { + if !reflect.DeepEqual(config, test.expected) { + fmt.Printf("") + t.Errorf(`Test: '%s' FAILED : expected %v, got %v`, name, test.expected, config) + } } } diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index eeefeb7c..fbcc9a26 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -9,9 +9,10 @@ import ( // HydrateConfig defines the hydrate function configurations, Name, Maximum number of concurrent calls to be allowed, dependencies type HydrateConfig struct { - Func HydrateFunc - MaxConcurrency int - RetryConfig *RetryConfig + Func HydrateFunc + MaxConcurrency int + RetryConfig *RetryConfig + ShouldIgnoreError ErrorPredicate Depends []HydrateFunc } diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index b896e66f..8440b54c 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -143,7 +143,7 @@ var testCasesValidate = map[string]validateTest{ }, RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}}, }, - expected: "table 'table' GetConfig does not specify a key", + expected: "table 'table' GetConfig does not specify a KeyColumn", }, "no get hydrate": { plugin: Plugin{ @@ -323,12 +323,13 @@ var testCasesValidate = map[string]validateTest{ }, RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}}, }, - expected: "table 'table' GetConfig does not specify a key", + expected: "table 'table' GetConfig does not specify a KeyColumn", }, } func TestValidate(t *testing.T) { for name, test := range testCasesValidate { + test.plugin.Initialise() validationErrors := test.plugin.Validate() if test.expected != validationErrors { diff --git a/plugin/table_test.go b/plugin/table_test.go index 85de7e2f..750e9bc4 100644 --- a/plugin/table_test.go +++ b/plugin/table_test.go @@ -10,8 +10,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v2/grpc/proto" ) -/////////////////////////////// -//// isGet +// isGet type isGetTest struct { table Table @@ -171,8 +170,7 @@ func TestIsGet(t *testing.T) { //} } -/////////////////////////////// -//// requiredHydrateCallBuilder +// requiredHydrateCallBuilder // declared in plugin_test // listHydrate, getHydrate, hydrate1, hydrate2, hydrate3, hydrate4 @@ -406,8 +404,7 @@ func hydrateCallToString(call *HydrateCall) string { return str } -/////////////////////////////// -//// getHydrateConfig +// getHydrateConfig type getHydrateConfigTest struct { table *Table @@ -565,29 +562,22 @@ func shouldIgnoreErrorTableDefault(err error) bool { func shouldIgnoreErrorPluginDefault(err error) bool { return true } -func shouldIgnoreError1(err error) bool { +func shouldIgnoreError1(error) bool { return true } -func shouldIgnoreError2(err error) bool { +func shouldIgnoreError2(error) bool { return true } -func shouldIgnoreError3(err error) bool { +func shouldRetryErrorTableDefault(error) bool { return true } -func shouldIgnoreError4(err error) bool { +func shouldRetryErrorPluginDefault(error) bool { return true } - -func shouldRetryErrorTableDefault(err error) bool { - return true -} -func shouldRetryErrorPluginDefault(err error) bool { +func shouldRetryError1(error) bool { return true } -func shouldRetryError1(err error) bool { - return true -} -func shouldRetryError2(err error) bool { +func shouldRetryError2(error) bool { return true } @@ -601,3 +591,5 @@ func TestGetHydrateConfig(t *testing.T) { } } } + +// validate table diff --git a/plugin/table_validate.go b/plugin/table_validate.go index 58cc8596..df4ef6c4 100644 --- a/plugin/table_validate.go +++ b/plugin/table_validate.go @@ -81,20 +81,20 @@ func (t *Table) validateListAndGetConfig() []string { var validationErrors []string // either get or list must be defined if t.List == nil && t.Get == nil { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' does not have either Get config or List config - one of these must be provided", t.Name)) + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' does not have either GetConfig or ListConfig - one of these must be provided", t.Name)) } if t.Get != nil { if t.Get.Hydrate == nil { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' Get config does not specify a hydrate function", t.Name)) + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' GetConfig does not specify a hydrate function", t.Name)) } if t.Get.KeyColumns == nil { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' Get config does not specify a KeyColumn", t.Name)) + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' GetConfig does not specify a KeyColumn", t.Name)) } } if t.List != nil { if t.List.Hydrate == nil { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' List config does not specify a hydrate function", t.Name)) + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' ListConfig does not specify a hydrate function", t.Name)) } } From e6fdf785d1854e2c0e95729cd30f02afdb521608 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 12:03:12 +0000 Subject: [PATCH 04/15] fix workflow --- .github/workflows/acceptance-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 4fe4b774..5ac47711 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -16,9 +16,9 @@ jobs: go-version: 1.16 - name: Run CLI Unit Tests - run: | - go clean -testcache - go test -timeout 30s ./... + run: | + go clean -testcache + go test -timeout 30s ./... - name: Checkout Steampipe uses: actions/checkout@v2 From c312e0145fa3c95f0191798bcb3fb880a4beb3d7 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 12:05:52 +0000 Subject: [PATCH 05/15] fix workflow --- .github/workflows/acceptance-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 5ac47711..7e62dbb3 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -10,12 +10,15 @@ jobs: name: Build Chaos Plugin runs-on: ubuntu-latest steps: + name: Checkout + uses: actions/checkout@v3 + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.16 - - name: Run CLI Unit Tests + - name: Run sdk Unit Tests run: | go clean -testcache go test -timeout 30s ./... From 83e8eaee488acf99c00e40a8b17eed44f046ba72 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 12:08:19 +0000 Subject: [PATCH 06/15] fix workflow --- .github/workflows/acceptance-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 7e62dbb3..16016d70 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -10,7 +10,7 @@ jobs: name: Build Chaos Plugin runs-on: ubuntu-latest steps: - name: Checkout + - name: Checkout uses: actions/checkout@v3 - name: Set up Go From 4124a742701083f697beafb7d7940a022e1aca22 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 12:12:03 +0000 Subject: [PATCH 07/15] force test run --- plugin/plugin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 8440b54c..c0a191b9 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -333,7 +333,7 @@ func TestValidate(t *testing.T) { validationErrors := test.plugin.Validate() if test.expected != validationErrors { - t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s'", name, test.expected, validationErrors) + t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s' ", name, test.expected, validationErrors) } } } From 8a493d0ae14a3cdf0c812c46a057b97c731f7d28 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 12:19:36 +0000 Subject: [PATCH 08/15] update acc WF --- .github/workflows/acceptance-test.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 16016d70..34c7e9d0 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -6,23 +6,33 @@ env: STEAMPIPE_UPDATE_CHECK: false jobs: + SDK unit tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Run SDK Unit Tests + run: | + go clean -testcache + go test -timeout 30s ./... + + buildChaosPlugin: name: Build Chaos Plugin runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v3 - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.16 - - name: Run sdk Unit Tests - run: | - go clean -testcache - go test -timeout 30s ./... - - name: Checkout Steampipe uses: actions/checkout@v2 with: From df099b3dc1d8401f904da71bd0e977cbb00109dc Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 12:24:12 +0000 Subject: [PATCH 09/15] force acc test --- plugin/plugin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index c0a191b9..8440b54c 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -333,7 +333,7 @@ func TestValidate(t *testing.T) { validationErrors := test.plugin.Validate() if test.expected != validationErrors { - t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s' ", name, test.expected, validationErrors) + t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s'", name, test.expected, validationErrors) } } } From 8e589ba3a4cad0c37aeb3362bf5ef21fc5694400 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Mar 2022 13:00:12 +0000 Subject: [PATCH 10/15] acc --- plugin/plugin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 8440b54c..478ac3dc 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -333,7 +333,7 @@ func TestValidate(t *testing.T) { validationErrors := test.plugin.Validate() if test.expected != validationErrors { - t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s'", name, test.expected, validationErrors) + t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s' ", name, test.expected, validationErrors) } } } From 1c4998a07f061381db283e437d148f73ee3936f9 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 28 Mar 2022 09:54:58 +0100 Subject: [PATCH 11/15] acc --- plugin/plugin_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 478ac3dc..aaa98fb4 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -35,6 +35,9 @@ func hydrate3(context.Context, *QueryData, *HydrateData) (interface{}, error) { func hydrate4(context.Context, *QueryData, *HydrateData) (interface{}, error) { return nil, nil } +func hydrate5(context.Context, *QueryData, *HydrateData) (interface{}, error) { + return nil, nil +} var testCasesValidate = map[string]validateTest{ "valid": { From 39fc84a1e8bec4dd3d6ea055d6b42e35d6de5002 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 28 Mar 2022 09:57:13 +0100 Subject: [PATCH 12/15] acc --- .github/workflows/acceptance-test.yml | 2 +- plugin/plugin_test.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 34c7e9d0..4b6fd75f 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -6,7 +6,7 @@ env: STEAMPIPE_UPDATE_CHECK: false jobs: - SDK unit tests: + sdkUnitTests: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index aaa98fb4..478ac3dc 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -35,9 +35,6 @@ func hydrate3(context.Context, *QueryData, *HydrateData) (interface{}, error) { func hydrate4(context.Context, *QueryData, *HydrateData) (interface{}, error) { return nil, nil } -func hydrate5(context.Context, *QueryData, *HydrateData) (interface{}, error) { - return nil, nil -} var testCasesValidate = map[string]validateTest{ "valid": { From 77f1d9b1aea9bc16932638baa96156c1e60be1b2 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 28 Mar 2022 10:27:54 +0100 Subject: [PATCH 13/15] acc --- plugin/plugin_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 478ac3dc..60e03e4f 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -2,6 +2,7 @@ package plugin import ( "context" + "log" "testing" "github.com/turbot/steampipe-plugin-sdk/v2/grpc/proto" @@ -328,10 +329,16 @@ var testCasesValidate = map[string]validateTest{ } func TestValidate(t *testing.T) { + log.Printf("[WARN] TestValidate\n") + for name, test := range testCasesValidate { + log.Printf("[WARN] %s\n", name) + test.plugin.Initialise() validationErrors := test.plugin.Validate() - + if len(validationErrors) > 0 { + log.Printf("[WARN] %s\n", validationErrors) + } if test.expected != validationErrors { t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s' ", name, test.expected, validationErrors) } From 699c3e92fbc0f399bb7be4129c5447d94319f6ec Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 28 Mar 2022 10:48:01 +0100 Subject: [PATCH 14/15] acc --- plugin/plugin_test.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 60e03e4f..478ac3dc 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -2,7 +2,6 @@ package plugin import ( "context" - "log" "testing" "github.com/turbot/steampipe-plugin-sdk/v2/grpc/proto" @@ -329,16 +328,10 @@ var testCasesValidate = map[string]validateTest{ } func TestValidate(t *testing.T) { - log.Printf("[WARN] TestValidate\n") - for name, test := range testCasesValidate { - log.Printf("[WARN] %s\n", name) - test.plugin.Initialise() validationErrors := test.plugin.Validate() - if len(validationErrors) > 0 { - log.Printf("[WARN] %s\n", validationErrors) - } + if test.expected != validationErrors { t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s' ", name, test.expected, validationErrors) } From 581cb6d1f47a931f1463ccb27a5bd3f782a44ff1 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 28 Mar 2022 11:02:49 +0100 Subject: [PATCH 15/15] comment --- plugin/plugin.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/plugin.go b/plugin/plugin.go index 94bde002..eaee3705 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -39,7 +39,6 @@ type Plugin struct { // (connection config is not available at plugin creation time) TableMapFunc func(ctx context.Context, p *Plugin) (map[string]*Table, error) - // DefaultTransform *transform.ColumnTransforms DefaultGetConfig *GetConfig DefaultConcurrency *DefaultConcurrencyConfig