Add table and plugin level defaults for ShouldIgnoreError and RetryConfig. Closes #257

.github/workflows/acceptance-test.yml

@@ -6,10 +6,28 @@ env:
   STEAMPIPE_UPDATE_CHECK: false
 
 jobs:
+  sdkUnitTests:
+    runs-on: ubuntu-latest
+    steps:
+        - name: Checkout
+          uses: actions/checkout@v3
+
+        - name: Set up Go
+          uses: actions/setup-go@v2
+          with:
+            go-version: 1.16
+
+        - name: Run SDK Unit Tests
+          run: |
+            go clean -testcache
+            go test -timeout 30s ./...
+
+
   buildChaosPlugin:
     name: Build Chaos Plugin
     runs-on: ubuntu-latest
     steps:
+
       - name: Set up Go
         uses: actions/setup-go@v2
         with:

cache/index_item.go

@@ -86,7 +86,7 @@ func (i IndexItem) SatisfiesQuals(checkQualMap map[string]*proto.Quals, keyColum
 			// isSubset means all data returned by check quals is returned by index quals
 			isSubset = checkQuals.IsASubsetOf(indexQuals)
 		} else {
-			log.Printf("[TRACE] SatisfiesQuals index item has qual for %s which check quals do not - NOT SATISFIED")
+			log.Printf("[TRACE] SatisfiesQuals index item has qual for %s which check quals do not - NOT SATISFIED", col)
 		}
 		log.Printf("[TRACE] get check qual %v, isSubset %v", ok, isSubset)
 		if !ok || !isSubset {

cache/query_cache.go

@@ -307,7 +307,7 @@ func (c *QueryCache) logMetrics() {
 	log.Printf("[TRACE] ------------------------------------ ")
 	log.Printf("[TRACE] Cache Metrics ")
 	log.Printf("[TRACE] ------------------------------------ ")
-	log.Printf("[TRACE] MaxCost: %d", c.cache.MaxCost)
+	log.Printf("[TRACE] MaxCost: %d", c.cache.MaxCost())
 	log.Printf("[TRACE] KeysAdded: %d", c.cache.Metrics.KeysAdded())
 	log.Printf("[TRACE] CostAdded: %d", c.cache.Metrics.CostAdded())
 	log.Printf("[TRACE] KeysEvicted: %d", c.cache.Metrics.KeysEvicted())

grpc/proto/qual.go

@@ -3,12 +3,12 @@ package proto
 import "log"
 
 func (x *Qual) Equals(other *Qual) bool {
-	log.Printf("[TRACE] me %s, other %s", x.String(), other.String())
+	//log.Printf("[TRACE] me %s, other %s", x.String(), other.String())
 	return x.String() == other.String()
 }
 
 func (x *Qual) IsASubsetOf(other *Qual) bool {
-	log.Printf("[TRACE] IsASubsetOf me %+v, other %+v", x, other)
+	//log.Printf("[TRACE] IsASubsetOf me %+v, other %+v", x, other)
 	operator, ok := x.Operator.(*Qual_StringValue)
 	if !ok {
 		log.Printf("[TRACE] IsASubsetOf my operator is not a string - returning false")
@@ -23,6 +23,11 @@ func (x *Qual) IsASubsetOf(other *Qual) bool {
 		log.Printf("[TRACE] IsASubsetOf Value nil - returning false")
 		return false
 	}
+	// check the fields are the same
+	if x.FieldName != other.FieldName {
+		log.Printf("[TRACE] IsASubsetOf field names different - returning false")
+		return false
+	}
 
 	switch value := x.Value.Value.(type) {
 	case *QualValue_StringValue:

plugin/concurrency.go

@@ -5,6 +5,14 @@ import (
 	"sync"
 )
 
+// DefaultConcurrencyConfig contains plugin level config to define default hydrate concurrency
+// - this is used if no HydrateConfig is specified for a specific call
+type DefaultConcurrencyConfig struct {
+	// max number of ALL hydrate calls in progress
+	TotalMaxConcurrency   int
+	DefaultMaxConcurrency int
+}
+
 // ConcurrencyManager struct ensures that hydrate functions stay within concurrency limits
 type ConcurrencyManager struct {
 	mut sync.Mutex

plugin/connection_config_test.go

@@ -106,6 +106,9 @@ var testCasesParseConfig = map[string]parseConfigTest{
 					Type:     schema.TypeString,
 					Required: true,
 				},
+				"count": {
+					Type: schema.TypeInt,
+				},
 			},
 		},
 		expectedFunc: func(res interface{}) bool {
@@ -438,12 +441,15 @@ func TestParseConnectionConfig(t *testing.T) {
 			}
 			continue
 		}
-		if test.expectedFunc != nil && !test.expectedFunc(config) {
-			t.Errorf(`Test: '%s' FAILED : expect verification func failed`, name)
-		}
-		if !reflect.DeepEqual(config, test.expected) {
-			fmt.Printf("")
-			t.Errorf(`Test: '%s' FAILED : expected %v, got %v`, name, test.expected, config)
+		if test.expectedFunc != nil {
+			if !test.expectedFunc(config) {
+				t.Errorf(`Test: '%s' FAILED : expect verification func failed`, name)
+			}
+		} else {
+			if !reflect.DeepEqual(config, test.expected) {
+				fmt.Printf("")
+				t.Errorf(`Test: '%s' FAILED : expected %v, got %v`, name, test.expected, config)
+			}
 		}
 
 	}

plugin/hydrate_call.go

@@ -6,45 +6,6 @@ import (
 	"github.com/turbot/go-kit/helpers"
 )
 
-// HydrateData contains the input data passed to every hydrate function
-type HydrateData struct {
-	// if there was a parent-child list call, store the parent list item
-	ParentItem     interface{}
-	Item           interface{}
-	HydrateResults map[string]interface{}
-}
-
-// HydrateFunc is a function which retrieves some or all row data for a single row item.
-type HydrateFunc func(context.Context, *QueryData, *HydrateData) (interface{}, error)
-
-// HydrateDependencies defines the hydrate function dependencies - other hydrate functions which must be run first
-// Deprecated: used HydrateConfig
-type HydrateDependencies struct {
-	Func    HydrateFunc
-	Depends []HydrateFunc
-}
-
-// HydrateConfig defines the hydrate function configurations, Name, Maximum number of concurrent calls to be allowed, dependencies
-type HydrateConfig struct {
-	Func              HydrateFunc
-	MaxConcurrency    int
-	RetryConfig       *RetryConfig
-	ShouldIgnoreError ErrorPredicate
-	Depends           []HydrateFunc
-}
-
-type RetryConfig struct {
-	ShouldRetryError ErrorPredicate
-}
-
-// DefaultConcurrencyConfig contains plugin level config to define default hydrate concurrency
-// - this is used if no HydrateConfig is specified for a specific call
-type DefaultConcurrencyConfig struct {
-	// max number of ALL hydrate calls in progress
-	TotalMaxConcurrency   int
-	DefaultMaxConcurrency int
-}
-
 // HydrateCall struct encapsulates a hydrate call, its config and dependencies
 type HydrateCall struct {
 	Func HydrateFunc

plugin/hydrate_config.go

@@ -0,0 +1,51 @@
+package plugin
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/turbot/go-kit/helpers"
+)
+
+// HydrateConfig defines the hydrate function configurations, Name, Maximum number of concurrent calls to be allowed, dependencies
+type HydrateConfig struct {
+	Func           HydrateFunc
+	MaxConcurrency int
+	RetryConfig    *RetryConfig
+
+	ShouldIgnoreError ErrorPredicate
+	Depends           []HydrateFunc
+}
+
+func (c *HydrateConfig) DefaultTo(defaultConfig *HydrateConfig) {
+	if defaultConfig == nil {
+		return
+	}
+	if c.RetryConfig == nil {
+		c.RetryConfig = defaultConfig.RetryConfig
+	}
+	if c.Depends == nil {
+		c.Depends = defaultConfig.Depends
+	}
+}
+
+func (c *HydrateConfig) String() interface{} {
+	shouldIgnoreErrorString := ""
+	if c.ShouldIgnoreError != nil {
+		shouldIgnoreErrorString = helpers.GetFunctionName(c.Func)
+	}
+	var dependsStrings = make([]string, len(c.Depends))
+	for i, dep := range c.Depends {
+		dependsStrings[i] = helpers.GetFunctionName(dep)
+	}
+	return fmt.Sprintf(`Func: %s
+MaxConcurrency: %d
+RetryConfig: %s
+ShouldIgnoreError: %s
+Depends: %s`,
+		helpers.GetFunctionName(c.Func),
+		c.MaxConcurrency,
+		c.RetryConfig.String(),
+		shouldIgnoreErrorString,
+		strings.Join(dependsStrings, ","))
+}

plugin/hydrate_data.go

@@ -0,0 +1,9 @@
+package plugin
+
+// HydrateData contains the input data passed to every hydrate function
+type HydrateData struct {
+	// if there was a parent-child list call, store the parent list item
+	ParentItem     interface{}
+	Item           interface{}
+	HydrateResults map[string]interface{}
+}

plugin/hydrate_dependencies.go

@@ -0,0 +1,8 @@
+package plugin
+
+// HydrateDependencies defines the hydrate function dependencies - other hydrate functions which must be run first
+// Deprecated: used HydrateConfig
+type HydrateDependencies struct {
+	Func    HydrateFunc
+	Depends []HydrateFunc
+}

plugin/hydrate_func.go

@@ -0,0 +1,8 @@
+package plugin
+
+import (
+	"context"
+)
+
+// HydrateFunc is a function which retrieves some or all row data for a single row item.
+type HydrateFunc func(context.Context, *QueryData, *HydrateData) (interface{}, error)

plugin/plugin.go

@@ -30,18 +30,20 @@ var validSchemaModes = []string{SchemaModeStatic, SchemaModeDynamic}
 
 // Plugin is an object used to build all necessary data for a given query
 type Plugin struct {
-	Name     string
-	Logger   hclog.Logger
+	Name   string
+	Logger hclog.Logger
+	// TableMap is a map of all the tables in the plugin, keyed by the table name
 	TableMap map[string]*Table
 	// TableMapFunc is a callback function which can be used to populate the table map
 	// this con optionally be provided by the plugin, and allows the connection config to be used in the table creation
 	// (connection config is not available at plugin creation time)
 	TableMapFunc func(ctx context.Context, p *Plugin) (map[string]*Table, error)
 
-	DefaultTransform   *transform.ColumnTransforms
-	DefaultGetConfig   *GetConfig
-	DefaultConcurrency *DefaultConcurrencyConfig
-	DefaultRetryConfig *RetryConfig
+	DefaultTransform         *transform.ColumnTransforms
+	DefaultGetConfig         *GetConfig
+	DefaultConcurrency       *DefaultConcurrencyConfig
+	DefaultRetryConfig       *RetryConfig
+	DefaultShouldIgnoreError ErrorPredicate
 	// every table must implement these columns
 	RequiredColumns        []*Column
 	ConnectionConfigSchema *ConnectionConfigSchema

plugin/plugin_test.go

@@ -143,7 +143,7 @@ var testCasesValidate = map[string]validateTest{
 			},
 			RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}},
 		},
-		expected: "table 'table' GetConfig does not specify a key",
+		expected: "table 'table' GetConfig does not specify a KeyColumn",
 	},
 	"no get hydrate": {
 		plugin: Plugin{
@@ -323,16 +323,17 @@ var testCasesValidate = map[string]validateTest{
 			},
 			RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}},
 		},
-		expected: "table 'table' GetConfig does not specify a key",
+		expected: "table 'table' GetConfig does not specify a KeyColumn",
 	},
 }
 
 func TestValidate(t *testing.T) {
 	for name, test := range testCasesValidate {
+		test.plugin.Initialise()
 		validationErrors := test.plugin.Validate()
 
 		if test.expected != validationErrors {
-			t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s'", name, test.expected, validationErrors)
+			t.Errorf("Test: '%s'' FAILED. \nExpected: '%s' \nGot: '%s'  ", name, test.expected, validationErrors)
 		}
 	}
 }

plugin/required_hydrate_calls.go

@@ -24,7 +24,8 @@ func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc) {
 
 	// if the resolved hydrate call is NOT the same as the fetch call, add to the map of hydrate functions to call
 	if hydrateName != c.fetchCallName {
-		if _, ok := c.requiredHydrateCalls[hydrateName]; !ok {
+		if _, ok := c.requiredHydrateCalls[hydrateName]; ok {
+			return
 		}
 
 		// get the config for this hydrate function

plugin/retry_config.go

@@ -0,0 +1,15 @@
+package plugin
+
+import (
+	"fmt"
+
+	"github.com/turbot/go-kit/helpers"
+)
+
+type RetryConfig struct {
+	ShouldRetryError ErrorPredicate
+}
+
+func (c RetryConfig) String() interface{} {
+	return fmt.Sprintf("ShouldRetryError: %s", helpers.GetFunctionName(c.ShouldRetryError))
+}

plugin/table.go

@@ -16,14 +16,18 @@ type Table struct {
 	// table description
 	Description string
 	// column definitions
-	Columns          []*Column
-	List             *ListConfig
-	Get              *GetConfig
-	GetMatrixItem    MatrixItemFunc
-	DefaultTransform *transform.ColumnTransforms
+	Columns                  []*Column
+	List                     *ListConfig
+	Get                      *GetConfig
+	GetMatrixItem            MatrixItemFunc
+	DefaultTransform         *transform.ColumnTransforms
+	DefaultShouldIgnoreError ErrorPredicate
+	DefaultRetryConfig       *RetryConfig
+
 	// the parent plugin object
 	Plugin *Plugin
 	// definitions of dependencies between hydrate functions
+	// Deprecated: used HydrateConfig
 	HydrateDependencies []HydrateDependencies
 	HydrateConfig       []HydrateConfig
 	// map of hydrate function name to columns it provides
@@ -97,10 +101,16 @@ func (t *Table) getFetchFunc(fetchType fetchType) HydrateFunc {
 		return t.List.Hydrate
 	}
 	return t.Get.Hydrate
-
 }
 
+// search through HydrateConfig and HydrateDependencies finding a function with the given name
+// if found return its dependencies
 func (t *Table) getHydrateDependencies(hydrateFuncName string) []HydrateFunc {
+	for _, d := range t.HydrateConfig {
+		if helpers.GetFunctionName(d.Func) == hydrateFuncName {
+			return d.Depends
+		}
+	}
 	for _, d := range t.HydrateDependencies {
 		if helpers.GetFunctionName(d.Func) == hydrateFuncName {
 			return d.Depends
@@ -115,11 +125,25 @@ func (t *Table) getHydrateConfig(hydrateFuncName string) *HydrateConfig {
 	for _, d := range t.HydrateConfig {
 		if helpers.GetFunctionName(d.Func) == hydrateFuncName {
 			config = &d
+			break
 		}
 	}
+	// now use default values if needed
 	if config.RetryConfig == nil {
-		config.RetryConfig = t.Plugin.DefaultRetryConfig
+		if t.DefaultRetryConfig != nil {
+			config.RetryConfig = t.DefaultRetryConfig
+		} else {
+			config.RetryConfig = t.Plugin.DefaultRetryConfig
+		}
+	}
+	if config.ShouldIgnoreError == nil {
+		if t.DefaultShouldIgnoreError != nil {
+			config.ShouldIgnoreError = t.DefaultShouldIgnoreError
+		} else {
+			config.ShouldIgnoreError = t.Plugin.DefaultShouldIgnoreError
+		}
 	}
+
 	// if no hydrate dependencies are specified in the hydrate config, check the deprecated "HydrateDependencies" property
 	if config.Depends == nil {
 		config.Depends = t.getHydrateDependencies(hydrateFuncName)