From a36dd1b6a4a78d6cbd086fd59a5b260244e03968 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 19 Sep 2023 09:50:50 +0100 Subject: [PATCH] Split IsRelationNotFoundError and GetMissingSchemaFromIsRelationNotFoundError and move to db_common rename rate steampipe_rate_limiter -> steampipe_plugin_limiter rename steampipe_connection_state -> steampipe_connection fix hcl range for all resources --- pkg/connection_sync/wait_for_search_path.go | 5 +- pkg/constants/db.go | 9 +-- pkg/db/db_client/db_client.go | 6 +- pkg/db/db_client/db_client_execute_retry.go | 31 +--------- pkg/db/db_common/errors.go | 38 +++++++++++++ pkg/db/db_local/internal.go | 13 +++-- .../interactive_client_autocomplete.go | 2 +- .../connection_state_table_sql.go | 57 +++++++++++++------ pkg/introspection/rate_limiters_table_sql.go | 4 +- pkg/pluginmanager_service/plugin_manager.go | 1 - .../plugin_manager_rate_limiters.go | 2 + pkg/steampipeconfig/connection_state.go | 20 +++++-- pkg/steampipeconfig/connection_state_map.go | 13 +++++ pkg/steampipeconfig/hclhelpers/hcl_blocks.go | 11 ++++ pkg/steampipeconfig/load_config.go | 3 +- pkg/steampipeconfig/load_connection_state.go | 22 +++++-- pkg/steampipeconfig/modconfig/benchmark.go | 3 +- pkg/steampipeconfig/modconfig/connection.go | 5 +- pkg/steampipeconfig/modconfig/control.go | 3 +- pkg/steampipeconfig/modconfig/dashboard.go | 3 +- .../modconfig/dashboard_card.go | 9 ++- .../modconfig/dashboard_category.go | 4 +- .../modconfig/dashboard_chart.go | 6 +- .../modconfig/dashboard_container.go | 7 ++- .../modconfig/dashboard_edge.go | 5 +- .../modconfig/dashboard_flow.go | 3 +- .../modconfig/dashboard_graph.go | 3 +- .../modconfig/dashboard_hierarchy.go | 6 +- .../modconfig/dashboard_image.go | 6 +- .../modconfig/dashboard_input.go | 6 +- .../modconfig/dashboard_node.go | 5 +- .../modconfig/dashboard_table.go | 8 +-- .../modconfig/dashboard_text.go | 8 +-- .../modconfig/dashboard_with.go | 5 +- pkg/steampipeconfig/modconfig/mod.go | 3 +- pkg/steampipeconfig/modconfig/param_def.go | 3 +- pkg/steampipeconfig/modconfig/plugin.go | 9 +-- .../modconfig/plugin_version.go | 3 +- pkg/steampipeconfig/modconfig/query.go | 3 +- pkg/steampipeconfig/modconfig/rate_limiter.go | 11 ++-- pkg/steampipeconfig/modconfig/require.go | 2 +- .../modconfig/steampipe_require.go | 2 +- .../modconfig/var_config/named_values.go | 3 +- .../modconfig/workspace_profile.go | 5 +- pkg/steampipeconfig/parse/connection.go | 4 +- pkg/steampipeconfig/parse/decode.go | 3 +- pkg/steampipeconfig/parse/decode_children.go | 3 +- pkg/steampipeconfig/parse/decode_options.go | 3 +- pkg/steampipeconfig/parse/mod.go | 6 +- .../parse/mod_parse_context_blocks.go | 3 +- pkg/steampipeconfig/parse/parse_context.go | 6 +- .../parse/workspace_profile.go | 5 +- 52 files changed, 255 insertions(+), 154 deletions(-) create mode 100644 pkg/db/db_common/errors.go diff --git a/pkg/connection_sync/wait_for_search_path.go b/pkg/connection_sync/wait_for_search_path.go index 7325295aed..38d3cd52fa 100644 --- a/pkg/connection_sync/wait_for_search_path.go +++ b/pkg/connection_sync/wait_for_search_path.go @@ -3,8 +3,6 @@ package connection_sync import ( "context" - "github.com/turbot/steampipe/pkg/constants" - "github.com/turbot/steampipe/pkg/db/db_client" "github.com/turbot/steampipe/pkg/db/db_common" "github.com/turbot/steampipe/pkg/steampipeconfig" ) @@ -24,8 +22,7 @@ func WaitForSearchPathSchemas(ctx context.Context, client db_common.Client, sear // NOTE: if we failed to load conection state, this must be because we are connected to an older version of the CLI // just return nil error - _, missingTable, relationNotFound := db_client.IsRelationNotFoundError(err) - if relationNotFound && missingTable == constants.ConnectionStateTable { + if db_common.IsRelationNotFoundError(err) { return nil } diff --git a/pkg/constants/db.go b/pkg/constants/db.go index 237ba30ffa..acfc0c6c62 100644 --- a/pkg/constants/db.go +++ b/pkg/constants/db.go @@ -54,12 +54,13 @@ const ( ServerSettingsTable = "steampipe_server_settings" // RateLimiterDefinitionTable is the table used to store rate limiters defined in the config - RateLimiterDefinitionTable = "steampipe_rate_limiter" + RateLimiterDefinitionTable = "steampipe_plugin_limiter" // PluginConfigTable is the table used to store plugin configs PluginConfigTable = "steampipe_plugin" - // ConnectionStateTable is the table used to store steampipe connection state - ConnectionStateTable = "steampipe_connection_state" + // LegacyConnectionStateTable is the table used to store steampipe connection state + LegacyConnectionStateTable = "steampipe_connection_state" + ConnectionTable = "steampipe_connection" ConnectionStatePending = "pending" ConnectionStatePendingIncomplete = "incomplete" ConnectionStateReady = "ready" @@ -94,7 +95,7 @@ const ( // ConnectionStates is a handy array of all states var ConnectionStates = []string{ - ConnectionStateTable, + LegacyConnectionStateTable, ConnectionStatePending, ConnectionStateReady, ConnectionStateUpdating, diff --git a/pkg/db/db_client/db_client.go b/pkg/db/db_client/db_client.go index fd5f46891d..32e8b83cf4 100644 --- a/pkg/db/db_client/db_client.go +++ b/pkg/db/db_client/db_client.go @@ -130,10 +130,10 @@ func (c *DbClient) closePools() { func (c *DbClient) loadServerSettings(ctx context.Context) error { serverSettings, err := serversettings.Load(ctx, c.managementPool) if err != nil { - if _, _, notFound := IsRelationNotFoundError(err); notFound { - // when connecting to pre-0.21.0 services, the server_settings table will not be available. + if notFound := db_common.IsRelationNotFoundError(err); notFound { + // when connecting to pre-0.21.0 services, the steampipe_server_settings table will not be available. // this is expected and not an error - // code which uses server_settings should handle this + // code which uses steampipe_server_settings should handle this log.Printf("[TRACE] could not find %s.%s table. skipping\n", constants.InternalSchema, constants.ServerSettingsTable) return nil } diff --git a/pkg/db/db_client/db_client_execute_retry.go b/pkg/db/db_client/db_client_execute_retry.go index d779f17581..b7a8be86ad 100644 --- a/pkg/db/db_client/db_client_execute_retry.go +++ b/pkg/db/db_client/db_client_execute_retry.go @@ -2,14 +2,11 @@ package db_client import ( "context" - "errors" "fmt" "log" - "regexp" "time" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/sethvargo/go-retry" typehelpers "github.com/turbot/go-kit/types" "github.com/turbot/steampipe/pkg/constants" @@ -47,7 +44,7 @@ func (c *DbClient) startQueryWithRetries(ctx context.Context, session *db_common log.Println("[TRACE] queryError:", queryError) // so there is an error - is it "relation not found"? - missingSchema, _, relationNotFound := IsRelationNotFoundError(queryError) + missingSchema, _, relationNotFound := db_common.GetMissingSchemaFromIsRelationNotFoundError(queryError) if !relationNotFound { log.Println("[TRACE] queryError not relation not found") // just return it @@ -138,29 +135,3 @@ func (c *DbClient) startQueryWithRetries(ctx context.Context, session *db_common return res, err } - -func IsRelationNotFoundError(err error) (string, string, bool) { - if err == nil { - return "", "", false - } - var pgErr *pgconn.PgError - ok := errors.As(err, &pgErr) - if !ok || pgErr.Code != "42P01" { - return "", "", false - } - - r := regexp.MustCompile(`^relation "(.*)\.(.*)" does not exist$`) - captureGroups := r.FindStringSubmatch(pgErr.Message) - if len(captureGroups) == 3 { - - return captureGroups[1], captureGroups[2], true - } - - // maybe there is no schema - r = regexp.MustCompile(`^relation "(.*)" does not exist$`) - captureGroups = r.FindStringSubmatch(pgErr.Message) - if len(captureGroups) == 2 { - return "", captureGroups[1], true - } - return "", "", true -} diff --git a/pkg/db/db_common/errors.go b/pkg/db/db_common/errors.go new file mode 100644 index 0000000000..e2288f55e9 --- /dev/null +++ b/pkg/db/db_common/errors.go @@ -0,0 +1,38 @@ +package db_common + +import ( + "errors" + "github.com/jackc/pgx/v5/pgconn" + "regexp" +) + +func IsRelationNotFoundError(err error) bool { + _, _, isRelationNotFound := GetMissingSchemaFromIsRelationNotFoundError(err) + return isRelationNotFound +} + +func GetMissingSchemaFromIsRelationNotFoundError(err error) (string, string, bool) { + if err == nil { + return "", "", false + } + var pgErr *pgconn.PgError + ok := errors.As(err, &pgErr) + if !ok || pgErr.Code != "42P01" { + return "", "", false + } + + r := regexp.MustCompile(`^relation "(.*)\.(.*)" does not exist$`) + captureGroups := r.FindStringSubmatch(pgErr.Message) + if len(captureGroups) == 3 { + + return captureGroups[1], captureGroups[2], true + } + + // maybe there is no schema + r = regexp.MustCompile(`^relation "(.*)" does not exist$`) + captureGroups = r.FindStringSubmatch(pgErr.Message) + if len(captureGroups) == 2 { + return "", captureGroups[1], true + } + return "", "", true +} diff --git a/pkg/db/db_local/internal.go b/pkg/db/db_local/internal.go index 43642aea70..57e97d93b1 100644 --- a/pkg/db/db_local/internal.go +++ b/pkg/db/db_local/internal.go @@ -9,7 +9,6 @@ import ( "github.com/jackc/pgx/v5" "github.com/turbot/steampipe-plugin-sdk/v5/sperr" "github.com/turbot/steampipe/pkg/constants" - "github.com/turbot/steampipe/pkg/db/db_client" "github.com/turbot/steampipe/pkg/db/db_common" "github.com/turbot/steampipe/pkg/introspection" "github.com/turbot/steampipe/pkg/statushooks" @@ -121,8 +120,8 @@ INNER JOIN "glob": true, } expectedTables := map[string]bool{ - "connection_state": true, // legacy table name - constants.ConnectionStateTable: true, + "connection_state": true, // previous legacy table name + constants.LegacyConnectionStateTable: true, } for _, f := range functions { @@ -234,18 +233,22 @@ func initializeConnectionStateTable(ctx context.Context, conn *pgx.Conn) error { connectionStateMap, err := steampipeconfig.LoadConnectionState(ctx, conn) if err != nil { // ignore relation not found error - _, _, isRelationNotFound := db_client.IsRelationNotFoundError(err) - if !isRelationNotFound { + if !db_common.IsRelationNotFoundError(err) { return err } + return err } // if any connections are in a ready state, set them to pending - we need to run refresh connections before we know this connection is still valid connectionStateMap.SetReadyConnectionsToPending() // if any connections are not in a ready or error state, set them to pending_incomplete connectionStateMap.SetNotReadyConnectionsToIncomplete() + // migration: ensure filename and line numbers are set for all connection states + connectionStateMap.PopulateFilename() + // drop the table and recreate queries := []db_common.QueryWithArgs{ + introspection.GetLegacyConnectionStateTableDropSql(), introspection.GetConnectionStateTableDropSql(), introspection.GetConnectionStateTableCreateSql(), introspection.GetConnectionStateTableGrantSql(), diff --git a/pkg/interactive/interactive_client_autocomplete.go b/pkg/interactive/interactive_client_autocomplete.go index 48f688b03e..7267c5ceaf 100644 --- a/pkg/interactive/interactive_client_autocomplete.go +++ b/pkg/interactive/interactive_client_autocomplete.go @@ -51,7 +51,7 @@ func (c *InteractiveClient) initialiseSchemaAndTableSuggestions(connectionStateM var unqualifiedTablesToAdd = make(map[string]struct{}) // add connection state and rate limit - unqualifiedTablesToAdd[constants.ConnectionStateTable] = struct{}{} + unqualifiedTablesToAdd[constants.ConnectionTable] = struct{}{} unqualifiedTablesToAdd[constants.RateLimiterDefinitionTable] = struct{}{} // get the first search path connection for each plugin diff --git a/pkg/introspection/connection_state_table_sql.go b/pkg/introspection/connection_state_table_sql.go index c77d604897..05740482a1 100644 --- a/pkg/introspection/connection_state_table_sql.go +++ b/pkg/introspection/connection_state_table_sql.go @@ -9,9 +9,14 @@ import ( "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) -// GetConnectionStateTableDropSql returns the sql to create the conneciton state table +// GetLegacyConnectionStateTableDropSql returns the sql to drop the legacy connection state table +func GetLegacyConnectionStateTableDropSql() db_common.QueryWithArgs { + query := fmt.Sprintf(`DROP TABLE IF EXISTS %s.%s;`, constants.InternalSchema, constants.LegacyConnectionStateTable) + return db_common.QueryWithArgs{Query: query} +} + func GetConnectionStateTableDropSql() db_common.QueryWithArgs { - query := fmt.Sprintf(`DROP TABLE IF EXISTS %s.%s;`, constants.InternalSchema, constants.ConnectionStateTable) + query := fmt.Sprintf(`DROP TABLE IF EXISTS %s.%s;`, constants.InternalSchema, constants.ConnectionTable) return db_common.QueryWithArgs{Query: query} } @@ -29,8 +34,11 @@ func GetConnectionStateTableCreateSql() db_common.QueryWithArgs { schema_hash TEXT NULL, comments_set BOOL DEFAULT FALSE, connection_mod_time TIMESTAMPTZ, - plugin_mod_time TIMESTAMPTZ -);`, constants.InternalSchema, constants.ConnectionStateTable) + plugin_mod_time TIMESTAMPTZ, + file_name TEXT, + start_line_number INTEGER, + end_line_number INTEGER +);`, constants.InternalSchema, constants.ConnectionTable) return db_common.QueryWithArgs{Query: query} } @@ -39,7 +47,7 @@ func GetConnectionStateTableGrantSql() db_common.QueryWithArgs { return db_common.QueryWithArgs{Query: fmt.Sprintf( `GRANT SELECT ON TABLE %s.%s TO %s;`, constants.InternalSchema, - constants.ConnectionStateTable, + constants.ConnectionTable, constants.DatabaseUsersRole, )} } @@ -53,7 +61,7 @@ SET state = '%s', WHERE name = $2 `, - constants.InternalSchema, constants.ConnectionStateTable, constants.ConnectionStateError) + constants.InternalSchema, constants.ConnectionTable, constants.ConnectionStateError) args := []any{constants.ConnectionStateError, err.Error(), connectionName} return db_common.QueryWithArgs{Query: query, Args: args} } @@ -69,7 +77,7 @@ WHERE AND state <> 'disabled' AND state <> 'error' `, - constants.InternalSchema, constants.ConnectionStateTable, constants.ConnectionStateError) + constants.InternalSchema, constants.ConnectionTable, constants.ConnectionStateError) args := []any{err.Error()} return db_common.QueryWithArgs{Query: query, Args: args} } @@ -89,8 +97,11 @@ func GetUpsertConnectionStateSql(c *steampipeconfig.ConnectionState) db_common.Q schema_hash, comments_set, connection_mod_time, - plugin_mod_time) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,now(),$12) + plugin_mod_time, + file_name, + start_line_number, + end_line_number) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,now(),$12,$13,$14,$15) ON CONFLICT (name) DO UPDATE SET @@ -105,9 +116,12 @@ DO schema_hash = $10, comments_set = $11, connection_mod_time = now(), - plugin_mod_time = $12 + plugin_mod_time = $12, + file_name = $13, + start_line_number = $14, + end_line_number = $15 -`, constants.InternalSchema, constants.ConnectionStateTable) +`, constants.InternalSchema, constants.ConnectionTable) args := []any{ c.ConnectionName, c.State, @@ -121,6 +135,9 @@ DO c.SchemaHash, c.CommentsSet, c.PluginModTime, + c.FileName, + c.StartLineNumber, + c.EndLineNumber, } return db_common.QueryWithArgs{Query: query, Args: args} } @@ -138,9 +155,12 @@ func GetNewConnectionStateFromConnectionInsertSql(c *modconfig.Connection) db_co schema_hash, comments_set, connection_mod_time, - plugin_mod_time) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,now(),now()) -`, constants.InternalSchema, constants.ConnectionStateTable) + plugin_mod_time, + file_name, + start_line_number, + end_line_number) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,now(),now(),$12,$13,$14) +`, constants.InternalSchema, constants.ConnectionTable) schemaMode := "" commentsSet := false @@ -157,6 +177,9 @@ VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,now(),now()) schemaMode, schemaHash, commentsSet, + c.DeclRange.Filename, + c.DeclRange.Start.Line, + c.DeclRange.End.Line, } return db_common.QueryWithArgs{ @@ -172,14 +195,14 @@ func GetSetConnectionStateSql(connectionName string, state string) db_common.Que WHERE name = $1 `, - constants.InternalSchema, constants.ConnectionStateTable, state, + constants.InternalSchema, constants.ConnectionTable, state, ) args := []any{connectionName} return db_common.QueryWithArgs{Query: query, Args: args} } func GetDeleteConnectionStateSql(connectionName string) db_common.QueryWithArgs { - query := fmt.Sprintf(`DELETE FROM %s.%s WHERE NAME=$1`, constants.InternalSchema, constants.ConnectionStateTable) + query := fmt.Sprintf(`DELETE FROM %s.%s WHERE NAME=$1`, constants.InternalSchema, constants.ConnectionTable) args := []any{connectionName} return db_common.QueryWithArgs{Query: query, Args: args} } @@ -187,7 +210,7 @@ func GetDeleteConnectionStateSql(connectionName string) db_common.QueryWithArgs func GetSetConnectionStateCommentLoadedSql(connectionName string, commentsLoaded bool) db_common.QueryWithArgs { query := fmt.Sprintf(`UPDATE %s.%s SET comments_set = $1 -WHERE NAME=$2`, constants.InternalSchema, constants.ConnectionStateTable) +WHERE NAME=$2`, constants.InternalSchema, constants.ConnectionTable) args := []any{commentsLoaded, connectionName} return db_common.QueryWithArgs{Query: query, Args: args} } diff --git a/pkg/introspection/rate_limiters_table_sql.go b/pkg/introspection/rate_limiters_table_sql.go index 1e53743858..75ad783c27 100644 --- a/pkg/introspection/rate_limiters_table_sql.go +++ b/pkg/introspection/rate_limiters_table_sql.go @@ -14,7 +14,7 @@ func GetRateLimiterTableCreateSql() db_common.QueryWithArgs { name TEXT, plugin TEXT, plugin_instance TEXT NULL, - source TEXT, + source_type TEXT, status TEXT, bucket_size INTEGER, fill_rate REAL , @@ -44,7 +44,7 @@ func GetRateLimiterTablePopulateSql(settings *modconfig.RateLimiter) db_common.Q "name", plugin, plugin_instance, -source, +source_type, status, bucket_size, fill_rate, diff --git a/pkg/pluginmanager_service/plugin_manager.go b/pkg/pluginmanager_service/plugin_manager.go index 0ae01ae43f..a8f2d63d37 100644 --- a/pkg/pluginmanager_service/plugin_manager.go +++ b/pkg/pluginmanager_service/plugin_manager.go @@ -77,7 +77,6 @@ type PluginManager struct { } func NewPluginManager(ctx context.Context, connectionConfig map[string]*sdkproto.ConnectionConfig, pluginConfigs connection.PluginMap, logger hclog.Logger) (*PluginManager, error) { - log.Printf("[INFO] NewPluginManager") pluginManager := &PluginManager{ logger: logger, diff --git a/pkg/pluginmanager_service/plugin_manager_rate_limiters.go b/pkg/pluginmanager_service/plugin_manager_rate_limiters.go index f81b2a3262..ab5dcd3ff7 100644 --- a/pkg/pluginmanager_service/plugin_manager_rate_limiters.go +++ b/pkg/pluginmanager_service/plugin_manager_rate_limiters.go @@ -211,6 +211,8 @@ func (m *PluginManager) initialiseRateLimiterDefs(ctx context.Context) (e error) if err != nil { return err } + // TODO KAI TACTICAL to force recreation + rateLimiterTableExists = false if !rateLimiterTableExists { return m.bootstrapRateLimiterTable(ctx) diff --git a/pkg/steampipeconfig/connection_state.go b/pkg/steampipeconfig/connection_state.go index 9c8c654595..82176956ea 100644 --- a/pkg/steampipeconfig/connection_state.go +++ b/pkg/steampipeconfig/connection_state.go @@ -14,10 +14,9 @@ import ( // ConnectionState is a struct containing all details for a connection // - the plugin name and checksum, the connection config and options // json tags needed as this is stored in the connection state file -// TODO KAI WHY THE omitempty type ConnectionState struct { // the connection name - ConnectionName string `json:"connection,omitempty" db:"name"` + ConnectionName string `json:"connection" db:"name"` // connection type (expected value: "aggregator") Type *string `json:"type,omitempty" db:"type"` // should we create a postgres schema for the connection (expected values: "enable", "disable") @@ -25,7 +24,7 @@ type ConnectionState struct { // the fully qualified name of the plugin Plugin string `json:"plugin" db:"plugin"` // the plugin instance - PluginInstance string `json:"plugin_instance,omitempty" db:"plugin_instance"` + PluginInstance string `json:"plugin_instance" db:"plugin_instance"` // the connection state (pending, updating, deleting, error, ready) State string `json:"state" db:"state"` // error (if there is one - make a pointer to support null) @@ -33,7 +32,7 @@ type ConnectionState struct { // schema mode - static or dynamic SchemaMode string `json:"schema_mode" db:"schema_mode"` // the hash of the connection schema - this is used to determine if a dynamic schema has changed - SchemaHash string `json:"schema_hash" db:"schema_hash"` + SchemaHash string `json:"schema_hash,omitempty" db:"schema_hash"` // are the comments set CommentsSet bool `json:"comments_set" db:"comments_set"` // the creation time of the plugin file @@ -41,7 +40,10 @@ type ConnectionState struct { // the update time of the connection ConnectionModTime time.Time `json:"connection_mod_time" db:"connection_mod_time"` // the matching patterns of child connections (for aggregators) - Connections []string `json:"connections" db:"connections"` + Connections []string `json:"connections" db:"connections"` + FileName string `json:"file_name" db:"file_name"` + StartLineNumber int `json:"start_line_number" db:"start_line_number"` + EndLineNumber int `json:"end_line_number" db:"end_line_number"` } func NewConnectionState(connection *modconfig.Connection, creationTime time.Time) *ConnectionState { @@ -55,10 +57,16 @@ func NewConnectionState(connection *modconfig.Connection, creationTime time.Time ImportSchema: connection.ImportSchema, Connections: connection.ConnectionNames, } - + state.setFilename(connection) return state } +func (d *ConnectionState) setFilename(connection *modconfig.Connection) { + d.FileName = connection.DeclRange.Filename + d.StartLineNumber = connection.DeclRange.Start.Line + d.EndLineNumber = connection.DeclRange.End.Line +} + func (d *ConnectionState) Equals(other *ConnectionState) bool { if d.Plugin != other.Plugin { return false diff --git a/pkg/steampipeconfig/connection_state_map.go b/pkg/steampipeconfig/connection_state_map.go index 21c30b7f0c..890cfa882b 100644 --- a/pkg/steampipeconfig/connection_state_map.go +++ b/pkg/steampipeconfig/connection_state_map.go @@ -236,3 +236,16 @@ func (m ConnectionStateMap) SetNotReadyConnectionsToIncomplete() { } } } + +// PopulateFilename sets the Filename, StartLineNumber and EndLineNumber properties +// this is required as these fields were added to the table after release +func (m ConnectionStateMap) PopulateFilename() { + // get the connection from config + connections := GlobalConfig.Connections + for name, state := range m { + // do we have config for this connection ( + if connection := connections[name]; connection != nil { + state.setFilename(connection) + } + } +} diff --git a/pkg/steampipeconfig/hclhelpers/hcl_blocks.go b/pkg/steampipeconfig/hclhelpers/hcl_blocks.go index 0eba6f0f13..753bd663d0 100644 --- a/pkg/steampipeconfig/hclhelpers/hcl_blocks.go +++ b/pkg/steampipeconfig/hclhelpers/hcl_blocks.go @@ -44,3 +44,14 @@ func BlocksToMap(blocks hcl.Blocks) map[string]*hcl.Block { } return res } + +func BlockRange(block *hcl.Block) hcl.Range { + if hclBody, ok := block.Body.(*hclsyntax.Body); ok { + return hclBody.SrcRange + } + return block.DefRange +} +func BlockRangePointer(block *hcl.Block) *hcl.Range { + r := BlockRange(block) + return &r +} diff --git a/pkg/steampipeconfig/load_config.go b/pkg/steampipeconfig/load_config.go index e862b5d85b..639d82eafc 100644 --- a/pkg/steampipeconfig/load_config.go +++ b/pkg/steampipeconfig/load_config.go @@ -18,6 +18,7 @@ import ( "github.com/turbot/steampipe/pkg/db/db_common" "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/filepaths" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "github.com/turbot/steampipe/pkg/steampipeconfig/options" "github.com/turbot/steampipe/pkg/steampipeconfig/parse" @@ -301,7 +302,7 @@ func loadConfig(configFolder string, steampipeConfig *SteampipeConfig, opts *loa diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagWarning, Summary: warning, - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }) } } diff --git a/pkg/steampipeconfig/load_connection_state.go b/pkg/steampipeconfig/load_connection_state.go index c94857cc38..6b7050fb68 100644 --- a/pkg/steampipeconfig/load_connection_state.go +++ b/pkg/steampipeconfig/load_connection_state.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/sethvargo/go-retry" "github.com/turbot/steampipe/pkg/constants" + "github.com/turbot/steampipe/pkg/db/db_common" "github.com/turbot/steampipe/pkg/filepaths" "github.com/turbot/steampipe/pkg/statushooks" "github.com/turbot/steampipe/pkg/utils" @@ -98,16 +99,31 @@ func loadConnectionState(ctx context.Context, conn *pgx.Conn, opts ...loadConnec } log.Println("[TRACE] with config", config) + var res = make(ConnectionStateMap) + query := fmt.Sprintf( `select * FROM %s.%s `, constants.InternalSchema, - constants.ConnectionStateTable, + constants.ConnectionTable, + ) + legacyQuery := fmt.Sprintf( + `select * FROM %s.%s `, + constants.InternalSchema, + constants.LegacyConnectionStateTable, ) rows, err := conn.Query(ctx, query) if err != nil { - return nil, err + if !db_common.IsRelationNotFoundError(err) { + return nil, err + } + // so it was a relation not found - try with legacy table + rows, err = conn.Query(ctx, legacyQuery) + if err != nil { + return nil, err + } } + defer rows.Close() connectionStateList, err := pgx.CollectRows(rows, pgx.RowToStructByNameLax[ConnectionState]) @@ -115,8 +131,6 @@ func loadConnectionState(ctx context.Context, conn *pgx.Conn, opts ...loadConnec return nil, err } - var res = make(ConnectionStateMap) - // convert to pointer arrau for _, c := range connectionStateList { // copy into loop var diff --git a/pkg/steampipeconfig/modconfig/benchmark.go b/pkg/steampipeconfig/modconfig/benchmark.go index 20bc19cd72..f16b31a510 100644 --- a/pkg/steampipeconfig/modconfig/benchmark.go +++ b/pkg/steampipeconfig/modconfig/benchmark.go @@ -2,6 +2,7 @@ package modconfig import ( "fmt" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/zclconf/go-cty/cty" "sort" "strings" @@ -40,7 +41,7 @@ func NewBenchmark(block *hcl.Block, mod *Mod, shortName string) HclResource { ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/connection.go b/pkg/steampipeconfig/modconfig/connection.go index 8b425b7338..3fd4842d1a 100644 --- a/pkg/steampipeconfig/modconfig/connection.go +++ b/pkg/steampipeconfig/modconfig/connection.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe/pkg/constants" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/options" "github.com/turbot/steampipe/pkg/utils" "golang.org/x/exp/maps" @@ -116,7 +117,7 @@ func NewPos(sourcePos hcl.Pos) Pos { func NewConnection(block *hcl.Block) *Connection { return &Connection{ Name: block.Labels[0], - DeclRange: NewRange(block.TypeRange), + DeclRange: NewRange(hclhelpers.BlockRange(block)), ImportSchema: ImportSchemaEnabled, } } @@ -151,7 +152,7 @@ func (c *Connection) SetOptions(opts options.Options, block *hcl.Block) hcl.Diag diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("invalid nested option type %s - only 'connection' options blocks are supported for Connections", reflect.TypeOf(o).Name()), - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }) } return diags diff --git a/pkg/steampipeconfig/modconfig/control.go b/pkg/steampipeconfig/modconfig/control.go index af7f53f9b4..7e10108c79 100644 --- a/pkg/steampipeconfig/modconfig/control.go +++ b/pkg/steampipeconfig/modconfig/control.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/turbot/go-kit/types" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" "github.com/zclconf/go-cty/cty" ) @@ -41,7 +42,7 @@ func NewControl(block *hcl.Block, mod *Mod, shortName string) HclResource { FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), ShortName: shortName, - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard.go b/pkg/steampipeconfig/modconfig/dashboard.go index c212810ab7..197a23743a 100644 --- a/pkg/steampipeconfig/modconfig/dashboard.go +++ b/pkg/steampipeconfig/modconfig/dashboard.go @@ -9,6 +9,7 @@ import ( "github.com/stevenle/topsort" typehelpers "github.com/turbot/go-kit/types" "github.com/turbot/steampipe/pkg/constants" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" "github.com/zclconf/go-cty/cty" ) @@ -46,7 +47,7 @@ func NewDashboard(block *hcl.Block, mod *Mod, shortName string) HclResource { ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_card.go b/pkg/steampipeconfig/modconfig/dashboard_card.go index bc7c5c23af..62e8553545 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_card.go +++ b/pkg/steampipeconfig/modconfig/dashboard_card.go @@ -3,12 +3,11 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" - - "github.com/turbot/steampipe/pkg/utils" - "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // DashboardCard is a struct representing a leaf dashboard node @@ -42,7 +41,7 @@ func NewDashboardCard(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_category.go b/pkg/steampipeconfig/modconfig/dashboard_category.go index d99410fc49..38a6e64704 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_category.go +++ b/pkg/steampipeconfig/modconfig/dashboard_category.go @@ -2,7 +2,9 @@ package modconfig import ( "fmt" + "github.com/hashicorp/hcl/v2" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" "github.com/zclconf/go-cty/cty" ) @@ -38,7 +40,7 @@ func NewDashboardCategory(block *hcl.Block, mod *Mod, shortName string) HclResou ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_chart.go b/pkg/steampipeconfig/modconfig/dashboard_chart.go index a29efc2342..ef590f3cf2 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_chart.go +++ b/pkg/steampipeconfig/modconfig/dashboard_chart.go @@ -3,11 +3,11 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // DashboardChart is a struct representing a leaf dashboard node @@ -41,7 +41,7 @@ func NewDashboardChart(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_container.go b/pkg/steampipeconfig/modconfig/dashboard_container.go index c7cc18cc9c..38bca0bd72 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_container.go +++ b/pkg/steampipeconfig/modconfig/dashboard_container.go @@ -2,12 +2,13 @@ package modconfig import ( "fmt" - typehelpers "github.com/turbot/go-kit/types" - "github.com/zclconf/go-cty/cty" "github.com/hashicorp/hcl/v2" "github.com/stevenle/topsort" + typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // TODO [node_reuse] add DashboardLeafNodeImpl @@ -38,7 +39,7 @@ func NewDashboardContainer(block *hcl.Block, mod *Mod, shortName string) HclReso ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_edge.go b/pkg/steampipeconfig/modconfig/dashboard_edge.go index 9b8dfd33fe..adfd5ac90e 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_edge.go +++ b/pkg/steampipeconfig/modconfig/dashboard_edge.go @@ -2,9 +2,10 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" "github.com/hashicorp/hcl/v2" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + "github.com/zclconf/go-cty/cty" ) // DashboardEdge is a struct representing a leaf dashboard node @@ -29,7 +30,7 @@ func NewDashboardEdge(block *hcl.Block, mod *Mod, shortName string) HclResource HclResourceImpl: HclResourceImpl{ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_flow.go b/pkg/steampipeconfig/modconfig/dashboard_flow.go index 9c21380a97..1e009c62ef 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_flow.go +++ b/pkg/steampipeconfig/modconfig/dashboard_flow.go @@ -5,6 +5,7 @@ import ( "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" "github.com/zclconf/go-cty/cty" ) @@ -44,7 +45,7 @@ func NewDashboardFlow(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_graph.go b/pkg/steampipeconfig/modconfig/dashboard_graph.go index 72b5d0c969..1f1041f5b7 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_graph.go +++ b/pkg/steampipeconfig/modconfig/dashboard_graph.go @@ -5,6 +5,7 @@ import ( "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" "github.com/zclconf/go-cty/cty" ) @@ -46,7 +47,7 @@ func NewDashboardGraph(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_hierarchy.go b/pkg/steampipeconfig/modconfig/dashboard_hierarchy.go index a4ee0d9ebf..00d7b876c4 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_hierarchy.go +++ b/pkg/steampipeconfig/modconfig/dashboard_hierarchy.go @@ -3,11 +3,11 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // DashboardHierarchy is a struct representing a leaf dashboard node @@ -46,7 +46,7 @@ func NewDashboardHierarchy(block *hcl.Block, mod *Mod, shortName string) HclReso ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_image.go b/pkg/steampipeconfig/modconfig/dashboard_image.go index 648f2c16e1..702604031d 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_image.go +++ b/pkg/steampipeconfig/modconfig/dashboard_image.go @@ -3,11 +3,11 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // DashboardImage is a struct representing a leaf dashboard node @@ -37,7 +37,7 @@ func NewDashboardImage(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fmt.Sprintf("%s.%s.%s", mod.ShortName, block.Type, shortName), UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_input.go b/pkg/steampipeconfig/modconfig/dashboard_input.go index b1b0ab8eae..b5c385c210 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_input.go +++ b/pkg/steampipeconfig/modconfig/dashboard_input.go @@ -3,11 +3,11 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // DashboardInput is a struct representing a leaf dashboard node @@ -45,7 +45,7 @@ func NewDashboardInput(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_node.go b/pkg/steampipeconfig/modconfig/dashboard_node.go index 2d640b3b34..862d1724d3 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_node.go +++ b/pkg/steampipeconfig/modconfig/dashboard_node.go @@ -2,9 +2,10 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" "github.com/hashicorp/hcl/v2" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + "github.com/zclconf/go-cty/cty" ) // DashboardNode is a struct representing a leaf dashboard node @@ -29,7 +30,7 @@ func NewDashboardNode(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_table.go b/pkg/steampipeconfig/modconfig/dashboard_table.go index 3362f3d861..2d95e64aec 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_table.go +++ b/pkg/steampipeconfig/modconfig/dashboard_table.go @@ -3,11 +3,11 @@ package modconfig import ( "fmt" - "github.com/turbot/steampipe/pkg/utils" - "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) const SnapshotQueryTableName = "custom.table.results" @@ -40,7 +40,7 @@ func NewDashboardTable(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_text.go b/pkg/steampipeconfig/modconfig/dashboard_text.go index 8f007663c3..ece7518949 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_text.go +++ b/pkg/steampipeconfig/modconfig/dashboard_text.go @@ -2,12 +2,12 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" - - "github.com/turbot/steampipe/pkg/utils" "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + "github.com/turbot/steampipe/pkg/utils" + "github.com/zclconf/go-cty/cty" ) // DashboardText is a struct representing a leaf dashboard node @@ -36,7 +36,7 @@ func NewDashboardText(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/dashboard_with.go b/pkg/steampipeconfig/modconfig/dashboard_with.go index 773d807090..66a0688410 100644 --- a/pkg/steampipeconfig/modconfig/dashboard_with.go +++ b/pkg/steampipeconfig/modconfig/dashboard_with.go @@ -2,9 +2,10 @@ package modconfig import ( "fmt" - "github.com/zclconf/go-cty/cty" "github.com/hashicorp/hcl/v2" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + "github.com/zclconf/go-cty/cty" ) // DashboardWith is a struct representing a leaf dashboard node @@ -26,7 +27,7 @@ func NewDashboardWith(block *hcl.Block, mod *Mod, shortName string) HclResource ShortName: shortName, FullName: fmt.Sprintf("%s.%s.%s", mod.ShortName, block.Type, shortName), UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/mod.go b/pkg/steampipeconfig/modconfig/mod.go index 345756379c..5cfc3097b0 100644 --- a/pkg/steampipeconfig/modconfig/mod.go +++ b/pkg/steampipeconfig/modconfig/mod.go @@ -13,6 +13,7 @@ import ( filehelpers "github.com/turbot/go-kit/files" typehelpers "github.com/turbot/go-kit/types" "github.com/turbot/steampipe/pkg/filepaths" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/zclconf/go-cty/cty" ) @@ -158,7 +159,7 @@ func (m *Mod) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostic return hcl.Diagnostics{&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Both 'require' and legacy 'requires' blocks are defined", - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }} } } diff --git a/pkg/steampipeconfig/modconfig/param_def.go b/pkg/steampipeconfig/modconfig/param_def.go index 3aca25ac41..ac804968ab 100644 --- a/pkg/steampipeconfig/modconfig/param_def.go +++ b/pkg/steampipeconfig/modconfig/param_def.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl/v2" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" ) type ParamDef struct { @@ -25,7 +26,7 @@ func NewParamDef(block *hcl.Block) *ParamDef { return &ParamDef{ ShortName: block.Labels[0], UnqualifiedName: fmt.Sprintf("param.%s", block.Labels[0]), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), } } diff --git a/pkg/steampipeconfig/modconfig/plugin.go b/pkg/steampipeconfig/modconfig/plugin.go index bb52923aea..aa5bec6009 100644 --- a/pkg/steampipeconfig/modconfig/plugin.go +++ b/pkg/steampipeconfig/modconfig/plugin.go @@ -2,8 +2,8 @@ package modconfig import ( "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/turbot/steampipe/pkg/ociinstaller" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" ) type Plugin struct { @@ -34,9 +34,10 @@ func NewImplicitPlugin(connection *Connection) *Plugin { } func (l *Plugin) OnDecoded(block *hcl.Block) { - l.FileName = &block.DefRange.Filename - l.StartLineNumber = &block.Body.(*hclsyntax.Body).SrcRange.Start.Line - l.EndLineNumber = &block.Body.(*hclsyntax.Body).SrcRange.End.Line + pluginRange := hclhelpers.BlockRange(block) + l.FileName = &pluginRange.Filename + l.StartLineNumber = &pluginRange.Start.Line + l.EndLineNumber = &pluginRange.End.Line l.imageRef = ociinstaller.NewSteampipeImageRef(l.Source) l.Plugin = l.imageRef.DisplayImageRef() } diff --git a/pkg/steampipeconfig/modconfig/plugin_version.go b/pkg/steampipeconfig/modconfig/plugin_version.go index a1e6c7e52c..7f85e71b7a 100644 --- a/pkg/steampipeconfig/modconfig/plugin_version.go +++ b/pkg/steampipeconfig/modconfig/plugin_version.go @@ -7,6 +7,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/hashicorp/hcl/v2" "github.com/turbot/steampipe/pkg/ociinstaller" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" ) type PluginVersion struct { @@ -41,7 +42,7 @@ func (p *PluginVersion) String() string { // Initialise parses the version and name properties func (p *PluginVersion) Initialise(block *hcl.Block) hcl.Diagnostics { var diags hcl.Diagnostics - p.DeclRange = block.DefRange + p.DeclRange = hclhelpers.BlockRange(block) // handle deprecation warnings/errors if p.VersionString != "" { if p.MinVersionString != "" { diff --git a/pkg/steampipeconfig/modconfig/query.go b/pkg/steampipeconfig/modconfig/query.go index ed8f742e7f..d48cef637e 100644 --- a/pkg/steampipeconfig/modconfig/query.go +++ b/pkg/steampipeconfig/modconfig/query.go @@ -12,6 +12,7 @@ import ( "github.com/turbot/go-kit/types" typehelpers "github.com/turbot/go-kit/types" "github.com/turbot/steampipe/pkg/constants" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/utils" ) @@ -38,7 +39,7 @@ func NewQuery(block *hcl.Block, mod *Mod, shortName string) HclResource { ShortName: shortName, FullName: fullName, UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), blockType: block.Type, }, Mod: mod, diff --git a/pkg/steampipeconfig/modconfig/rate_limiter.go b/pkg/steampipeconfig/modconfig/rate_limiter.go index d46cf454ea..82f61a11a4 100644 --- a/pkg/steampipeconfig/modconfig/rate_limiter.go +++ b/pkg/steampipeconfig/modconfig/rate_limiter.go @@ -5,9 +5,9 @@ import ( "strings" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe/pkg/ociinstaller" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" ) const ( @@ -30,7 +30,7 @@ type RateLimiter struct { StartLineNumber *int `db:"start_line_number" json:"-"` EndLineNumber *int `db:"end_line_number" json:"-"` Status string `db:"status"` - Source string `db:"source"` + Source string `db:"source_type"` ImageRef *ociinstaller.SteampipeImageRef `db:"-"` } @@ -81,9 +81,10 @@ func (l *RateLimiter) AsProto() *proto.RateLimiterDefinition { } func (l *RateLimiter) OnDecoded(block *hcl.Block) { - l.FileName = &block.DefRange.Filename - l.StartLineNumber = &block.Body.(*hclsyntax.Body).SrcRange.Start.Line - l.EndLineNumber = &block.Body.(*hclsyntax.Body).SrcRange.End.Line + limiterRange := hclhelpers.BlockRange(block) + l.FileName = &limiterRange.Filename + l.StartLineNumber = &limiterRange.Start.Line + l.EndLineNumber = &limiterRange.End.Line if l.Scope == nil { l.Scope = []string{} } diff --git a/pkg/steampipeconfig/modconfig/require.go b/pkg/steampipeconfig/modconfig/require.go index d40010cb10..e7ef5f5adf 100644 --- a/pkg/steampipeconfig/modconfig/require.go +++ b/pkg/steampipeconfig/modconfig/require.go @@ -74,7 +74,7 @@ func (r *Require) initialise(modBlock *hcl.Block) hcl.Diagnostics { } // set our Ranges - r.DeclRange = requireBlock.DefRange + r.DeclRange = hclhelpers.BlockRange(requireBlock) r.BodyRange = requireBlock.Body.(*hclsyntax.Body).SrcRange // build maps of plugin and mod blocks diff --git a/pkg/steampipeconfig/modconfig/steampipe_require.go b/pkg/steampipeconfig/modconfig/steampipe_require.go index c3572a5688..2ad2cb22b0 100644 --- a/pkg/steampipeconfig/modconfig/steampipe_require.go +++ b/pkg/steampipeconfig/modconfig/steampipe_require.go @@ -23,7 +23,7 @@ func (r *SteampipeRequire) initialise(requireBlock *hcl.Block) hcl.Diagnostics { steampipeBlock = requireBlock } // set DeclRange - r.DeclRange = steampipeBlock.DefRange + r.DeclRange = hclhelpers.BlockRange(steampipeBlock) if r.MinVersionString == "" { return nil diff --git a/pkg/steampipeconfig/modconfig/var_config/named_values.go b/pkg/steampipeconfig/modconfig/var_config/named_values.go index 5c5d826663..27bc6b5389 100644 --- a/pkg/steampipeconfig/modconfig/var_config/named_values.go +++ b/pkg/steampipeconfig/modconfig/var_config/named_values.go @@ -3,6 +3,7 @@ package var_config // github.com/hashicorp/terraform/configs/parser_config.go import ( "fmt" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "unicode" "github.com/hashicorp/hcl/v2" @@ -36,7 +37,7 @@ type Variable struct { func DecodeVariableBlock(block *hcl.Block, content *hcl.BodyContent, override bool) (*Variable, hcl.Diagnostics) { v := &Variable{ Name: block.Labels[0], - DeclRange: block.DefRange, + DeclRange: hclhelpers.BlockRange(block), } var diags hcl.Diagnostics diff --git a/pkg/steampipeconfig/modconfig/workspace_profile.go b/pkg/steampipeconfig/modconfig/workspace_profile.go index cd9a7c01fe..dcbcecf0e2 100644 --- a/pkg/steampipeconfig/modconfig/workspace_profile.go +++ b/pkg/steampipeconfig/modconfig/workspace_profile.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/spf13/cobra" "github.com/turbot/steampipe/pkg/constants" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/options" "github.com/zclconf/go-cty/cty" ) @@ -71,7 +72,7 @@ func (p *WorkspaceProfile) SetOptions(opts options.Options, block *hcl.Block) hc diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("invalid nested option type %s - only 'connection' options blocks are supported for Connections", reflect.TypeOf(o).Name()), - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }) } return diags @@ -81,7 +82,7 @@ func duplicateOptionsBlockDiag(block *hcl.Block) *hcl.Diagnostic { return &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("duplicate %s options block", block.Type), - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), } } diff --git a/pkg/steampipeconfig/parse/connection.go b/pkg/steampipeconfig/parse/connection.go index 4360790b5d..d6fb88a4cc 100644 --- a/pkg/steampipeconfig/parse/connection.go +++ b/pkg/steampipeconfig/parse/connection.go @@ -76,7 +76,7 @@ func DecodeConnection(block *hcl.Block) (*modconfig.Connection, hcl.Diagnostics) diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagWarning, Summary: fmt.Sprintf("%s in %s have been deprecated and will be removed in subsequent versions of steampipe", constants.Bold("'connection' options"), constants.Bold("'connection' blocks")), - Subject: &connectionBlock.DefRange, + Subject: hclhelpers.BlockRangePointer(connectionBlock), }) } @@ -85,7 +85,7 @@ func DecodeConnection(block *hcl.Block) (*modconfig.Connection, hcl.Diagnostics) diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("invalid block type '%s' - only 'options' blocks are supported for Connections", connectionBlock.Type), - Subject: &connectionBlock.DefRange, + Subject: hclhelpers.BlockRangePointer(connectionBlock), }) } } diff --git a/pkg/steampipeconfig/parse/decode.go b/pkg/steampipeconfig/parse/decode.go index 2c5054b2bb..450a565980 100644 --- a/pkg/steampipeconfig/parse/decode.go +++ b/pkg/steampipeconfig/parse/decode.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config" ) @@ -232,7 +233,7 @@ func resourceForBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.Hc return nil, hcl.Diagnostics{&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("resourceForBlock called for unsupported block type %s", block.Type), - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }, } } diff --git a/pkg/steampipeconfig/parse/decode_children.go b/pkg/steampipeconfig/parse/decode_children.go index 08ed61e350..9a13cd235c 100644 --- a/pkg/steampipeconfig/parse/decode_children.go +++ b/pkg/steampipeconfig/parse/decode_children.go @@ -5,6 +5,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) @@ -69,7 +70,7 @@ func checkForDuplicateChildren(names []string, block *hcl.Block) hcl.Diagnostics diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("'%s.%s' has duplicate child name '%s'", block.Type, block.Labels[0], n), - Subject: &block.DefRange}) + Subject: hclhelpers.BlockRangePointer(block)}) } nameMap[n] = nameCount + 1 } diff --git a/pkg/steampipeconfig/parse/decode_options.go b/pkg/steampipeconfig/parse/decode_options.go index 800b3ec139..51a289f293 100644 --- a/pkg/steampipeconfig/parse/decode_options.go +++ b/pkg/steampipeconfig/parse/decode_options.go @@ -5,6 +5,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/options" ) @@ -21,7 +22,7 @@ func DecodeOptions(block *hcl.Block, overrides ...BlockMappingOverride) (options diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("Unexpected options type '%s'", block.Labels[0]), - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }) return nil, diags } diff --git a/pkg/steampipeconfig/parse/mod.go b/pkg/steampipeconfig/parse/mod.go index c541fbcb21..ec8c84ec09 100644 --- a/pkg/steampipeconfig/parse/mod.go +++ b/pkg/steampipeconfig/parse/mod.go @@ -7,7 +7,6 @@ import ( "path" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/filepaths" @@ -77,10 +76,7 @@ func ParseModDefinition(modPath string, evalCtx *hcl.EvalContext) (*modconfig.Mo }) return nil, res } - var defRange = block.DefRange - if hclBody, ok := block.Body.(*hclsyntax.Body); ok { - defRange = hclBody.SrcRange - } + var defRange = hclhelpers.BlockRange(block) mod := modconfig.NewMod(block.Labels[0], path.Dir(modFilePath), defRange) // set modFilePath mod.SetFilePath(modFilePath) diff --git a/pkg/steampipeconfig/parse/mod_parse_context_blocks.go b/pkg/steampipeconfig/parse/mod_parse_context_blocks.go index fbfb7534d2..f308adc286 100644 --- a/pkg/steampipeconfig/parse/mod_parse_context_blocks.go +++ b/pkg/steampipeconfig/parse/mod_parse_context_blocks.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) @@ -67,7 +68,7 @@ func (m *ModParseContext) cacheBlockName(block *hcl.Block, shortName string) { } func (m *ModParseContext) blockHash(block *hcl.Block) string { - return helpers.GetMD5Hash(block.DefRange.String()) + return helpers.GetMD5Hash(hclhelpers.BlockRange(block).String()) } // getUniqueName returns a name unique within the scope of this execution tree diff --git a/pkg/steampipeconfig/parse/parse_context.go b/pkg/steampipeconfig/parse/parse_context.go index 87ba4fa959..e9c90dca76 100644 --- a/pkg/steampipeconfig/parse/parse_context.go +++ b/pkg/steampipeconfig/parse/parse_context.go @@ -5,7 +5,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/stevenle/topsort" "github.com/turbot/go-kit/helpers" - "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" + hclhelpers "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "github.com/zclconf/go-cty/cty" "strings" @@ -119,10 +119,10 @@ func (r *ParseContext) BlocksToDecode() (hcl.Blocks, error) { // depOrder is all the blocks required to resolve dependencies. // if this one is unparsed, added to list block, ok := r.UnresolvedBlocks[name] - if ok && !blocksMap[block.Block.DefRange.String()] { + if !blocksMap[hclhelpers.BlockRange(block.Block).String()] && ok { blocksToDecode = append(blocksToDecode, block.Block) // add to map - blocksMap[block.Block.DefRange.String()] = true + blocksMap[hclhelpers.BlockRange(block.Block).String()] = true } } return blocksToDecode, nil diff --git a/pkg/steampipeconfig/parse/workspace_profile.go b/pkg/steampipeconfig/parse/workspace_profile.go index 979054b2cd..62e039e985 100644 --- a/pkg/steampipeconfig/parse/workspace_profile.go +++ b/pkg/steampipeconfig/parse/workspace_profile.go @@ -10,6 +10,7 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe/pkg/constants" + "github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "github.com/turbot/steampipe/pkg/steampipeconfig/options" ) @@ -158,7 +159,7 @@ func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseCon // fail diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), Summary: fmt.Sprintf("Duplicate options type '%s'", optionsBlockType), }) } @@ -177,7 +178,7 @@ func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseCon diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("invalid block type '%s' - only 'options' blocks are supported for workspace profiles", block.Type), - Subject: &block.DefRange, + Subject: hclhelpers.BlockRangePointer(block), }) } }