feat: Upgrade to SDK v4 #7

maaarcelino · 2023-11-01T14:03:27Z

Hi there! This is a PR to update to our latest SDK! We are also rolling out hub shortly - https://hub.cloudquery.io so, once this PR is approved we can follow-up with additional PR so publish the plugin to our hub so users can enjoy dynamic searchable documentation for all versions and all tables.

candiduslynx · 2023-11-01T14:32:47Z

plugin/plugin.go

 )

 var (
+	Kind    = "source"
+	Team    = "coinpaprika"
+	Name    = "coinpaprika-coinpaprika"


Suggested change

Name = "coinpaprika-coinpaprika"

Name = "coinpaprika"

Done in f40bcbd

candiduslynx · 2023-11-01T14:33:53Z

resources/services/coins/tickers.go

@@ -52,7 +52,7 @@ func fetchTickers(ctx context.Context, meta schema.ClientMeta, parent *schema.Re
 	startDate := cl.StartDate
 	key := fmt.Sprintf(stateKeyTpl, *c.ID)
 	if cl.Backend != nil {
-		value, err := cl.Backend.Get(ctx, key, cl.ID())
+		value, err := cl.Backend.GetKey(ctx, key)


IMO you could just use state.NopClient & never perform the if cl.Backend != nil check

candiduslynx · 2023-11-01T14:34:18Z

resources/services/coins/tickers.go

-			return fmt.Errorf("set state failure: %w", err)
+			return fmt.Errorf("failed to save state to backend: %w", err)
+		}
+		err = cl.Backend.Flush(ctx)


no need to perform manual flush, done in plugin.Client.Sync

candiduslynx · 2023-11-01T14:34:33Z

plugin/client.go

+type Client struct {
+	logger      zerolog.Logger
+	config      client.Spec
+	backendConn *grpc.ClientConn


Suggested change

backendConn *grpc.ClientConn

Done in f40bcbd

candiduslynx · 2023-11-01T14:34:43Z

plugin/client.go

+	if c.backendConn != nil {
+		return c.backendConn.Close()
+	}


Suggested change

if c.backendConn != nil {

return c.backendConn.Close()

}

Done in f40bcbd

candiduslynx · 2023-11-01T14:35:18Z

plugin/client.go

+}
+
+func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error {
+	tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)


verify for no connection, see https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/resources/plugin/client.go

candiduslynx · 2023-11-01T14:36:59Z

plugin/client.go

+	var stateClient state.Client
+	if options.BackendOptions == nil {
+		c.logger.Info().Msg("No backend options provided, using no state backend")
+		stateClient = &state.NoOpClient{}
+		c.backendConn = nil
+	} else {
+		c.backendConn, err = grpc.DialContext(ctx, options.BackendOptions.Connection,
+			grpc.WithTransportCredentials(insecure.NewCredentials()),
+			grpc.WithDefaultCallOptions(
+				grpc.MaxCallRecvMsgSize(maxMsgSize),
+				grpc.MaxCallSendMsgSize(maxMsgSize),
+			),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to dial grpc source plugin at %s: %w", options.BackendOptions.Connection, err)
+		}
+		stateClient, err = state.NewClient(ctx, c.backendConn, options.BackendOptions.TableName)
+		if err != nil {
+			return fmt.Errorf("failed to create state client: %w", err)
+		}
+		c.logger.Info().Str("table_name", options.BackendOptions.TableName).Msg("Connected to state backend")
+	}


Suggested change

var stateClient state.Client

if options.BackendOptions == nil {

c.logger.Info().Msg("No backend options provided, using no state backend")

stateClient = &state.NoOpClient{}

c.backendConn = nil

} else {

c.backendConn, err = grpc.DialContext(ctx, options.BackendOptions.Connection,

grpc.WithTransportCredentials(insecure.NewCredentials()),

grpc.WithDefaultCallOptions(

grpc.MaxCallRecvMsgSize(maxMsgSize),

grpc.MaxCallSendMsgSize(maxMsgSize),

),

)

if err != nil {

return fmt.Errorf("failed to dial grpc source plugin at %s: %w", options.BackendOptions.Connection, err)

}

stateClient, err = state.NewClient(ctx, c.backendConn, options.BackendOptions.TableName)

if err != nil {

return fmt.Errorf("failed to create state client: %w", err)

}

c.logger.Info().Str("table_name", options.BackendOptions.TableName).Msg("Connected to state backend")

}

stateClient := state.Client(&state.NoOpClient{})

if options.BackendOptions != nil {

conn, err := grpc.DialContext(ctx, options.BackendOptions.Connection,

grpc.WithTransportCredentials(insecure.NewCredentials()),

grpc.WithDefaultCallOptions(

grpc.MaxCallRecvMsgSize(maxMsgSize),

grpc.MaxCallSendMsgSize(maxMsgSize),

),

)

if err != nil {

return fmt.Errorf("failed to dial grpc destination plugin at %s: %w", options.BackendOptions.Connection, err)

}

stateClient, err = state.NewClient(ctx, conn, options.BackendOptions.TableName)

if err != nil {

return fmt.Errorf("failed to create state client: %w", err)

}

}

Done in f40bcbd

candiduslynx · 2023-11-01T15:21:12Z

resources/services/coins/tickers.go

@@ -52,7 +52,7 @@ func fetchTickers(ctx context.Context, meta schema.ClientMeta, parent *schema.Re
 	startDate := cl.StartDate
 	key := fmt.Sprintf(stateKeyTpl, *c.ID)
 	if cl.Backend != nil {
-		value, err := cl.Backend.Get(ctx, key, cl.ID())
+		value, err := cl.Backend.GetKey(ctx, key)


candiduslynx · 2023-11-01T15:21:27Z

resources/services/coins/tickers.go

@@ -93,9 +93,9 @@ func fetchTickers(ctx context.Context, meta schema.ClientMeta, parent *schema.Re
 		res <- tt
 	}
 	if cl.Backend != nil {
-		err = cl.Backend.Set(ctx, key, cl.ID(), upTo.Format(time.RFC3339))
+		err = cl.Backend.SetKey(ctx, key, upTo.Format(time.RFC3339))


remove if cl.Backend != nil check

candiduslynx · 2023-11-01T16:09:29Z

cc: @mbillewicz @r--w

r--w · 2023-11-02T13:33:14Z

Wow, thanks for this contribution; it seems like you spent some time on this PR. We will review the code within a week.

r--w · 2023-11-02T13:48:24Z

Hey @candiduslynx & @maaarcelino could you fix the quality/linter issues?

You can run this command locally make check

maaarcelino · 2023-11-03T11:07:22Z

@r--w There's only a make lint command which passes for me locally without making any changes. Those logs appear to point to some other code (e.g. deleteValue is not present in any of the files in this repo).

There were a few failing tests due to a nil pointer and I've fixed those. Now both make lint and make test all pass.

mbillewicz · 2023-11-03T12:49:47Z

@maaarcelino Try to update linter in .github/workflows/test.yaml line 26 to latest version.

maaarcelino · 2023-11-03T13:21:37Z

@mbillewicz Updated to 1.55.1 and added skip for build/pkg folders as the linting included folders outside the project

mbillewicz · 2023-11-04T21:13:39Z

@maaarcelino pls also regenerate files by make gen due to failing workflow test job

r--w · 2023-11-06T12:35:41Z

@maaarcelino merged - thank you.

maaarcelino added 3 commits November 1, 2023 10:28

Upgrade cloudquery/plugin-sdk to v4 and remove v3

9bb51db

Update readme

9d56853

Update table schemas

4333e8c

candiduslynx suggested changes Nov 1, 2023

View reviewed changes

PR comments

f40bcbd

maaarcelino requested a review from candiduslynx November 1, 2023 14:55

candiduslynx suggested changes Nov 1, 2023

View reviewed changes

Remove backend nil checks

f50d602

candiduslynx approved these changes Nov 1, 2023

View reviewed changes

Fix tests

8022d52

Update linter to 1.55.1

bb0a6a2

Skip linting for pkg and cache

2bb778e

mbillewicz approved these changes Nov 3, 2023

View reviewed changes

Generate docs

b9c3d13

r--w merged commit d9af40e into coinpaprika:master Nov 6, 2023
1 check passed

maaarcelino mentioned this pull request Nov 8, 2023

feat: Add documentation and plugin publishing commands #8

Merged

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: Upgrade to SDK v4 #7

feat: Upgrade to SDK v4 #7

maaarcelino commented Nov 1, 2023

candiduslynx Nov 1, 2023

maaarcelino Nov 1, 2023

candiduslynx Nov 1, 2023

candiduslynx Nov 1, 2023

candiduslynx Nov 1, 2023

candiduslynx Nov 1, 2023

maaarcelino Nov 1, 2023

candiduslynx Nov 1, 2023

maaarcelino Nov 1, 2023

candiduslynx Nov 1, 2023

candiduslynx Nov 1, 2023

maaarcelino Nov 1, 2023

candiduslynx Nov 1, 2023

candiduslynx Nov 1, 2023

candiduslynx commented Nov 1, 2023

r--w commented Nov 2, 2023

r--w commented Nov 2, 2023

maaarcelino commented Nov 3, 2023

mbillewicz commented Nov 3, 2023 •

edited

Loading

maaarcelino commented Nov 3, 2023 •

edited

Loading

mbillewicz commented Nov 4, 2023

r--w commented Nov 6, 2023

feat: Upgrade to SDK v4 #7

feat: Upgrade to SDK v4 #7

Conversation

maaarcelino commented Nov 1, 2023

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

candiduslynx commented Nov 1, 2023

r--w commented Nov 2, 2023

r--w commented Nov 2, 2023

maaarcelino commented Nov 3, 2023

mbillewicz commented Nov 3, 2023 • edited Loading

maaarcelino commented Nov 3, 2023 • edited Loading

mbillewicz commented Nov 4, 2023

r--w commented Nov 6, 2023

mbillewicz commented Nov 3, 2023 •

edited

Loading

maaarcelino commented Nov 3, 2023 •

edited

Loading