Merge pull request #7 from maaarcelino/feat/upgrade-to-sdk-v4

feat: Upgrade to SDK v4

.github/workflows/test.yaml

@@ -23,8 +23,10 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.55.1
           args: --timeout=3m
+          skip-pkg-cache: true
+          skip-build-cache: true
       - name: Get dependencies
         run: go get -t -d ./...
       - name: Build

README.md

@@ -31,8 +31,10 @@ The following source configuration file will sync to a PostgreSQL database. See
      spec:
        name: "coinpaprika"
        path: "coinpaprika/coinpaprika"
-       version: "v1.0.1"
-       backend: local
+       version: "v2.0.0"
+       backend_options:
+         table_name: "cq_state_coinpaprika"
+         connection: "@@plugins.sqlite.connection"
        tables:
          [ "*" ]
        destinations:
@@ -50,7 +52,8 @@ The following source configuration file will sync to a PostgreSQL database. See
      spec:
        name: sqlite
        path: cloudquery/sqlite
-       version: "v2.2.1"
+       registry: cloudquery
+       version: "v2.4.15"
        spec:
          connection_string: ./db.sql    
      ```
@@ -62,8 +65,10 @@ The following source configuration file will sync to a PostgreSQL database. See
     spec:
       name: "coinpaprika"
       path: "coinpaprika/coinpaprika"
-      version: "v1.0.1"
-      backend: local
+      version: "v2.0.0"
+      backend_options:
+        table_name: "cq_state_coinpaprika"
+        connection: "@@plugins.sqlite.connection"
       tables:
         [ "*" ]
       destinations:
@@ -82,7 +87,8 @@ The following source configuration file will sync to a PostgreSQL database. See
     spec:
       name: sqlite
       path: cloudquery/sqlite
-      version: "v2.2.1"
+      registry: cloudquery
+      version: "v2.4.15"
       spec:
         connection_string: ./db.sql 
     ```

client/backend.go

@@ -1,7 +1,7 @@
 package client
 
-import "github.com/cloudquery/plugin-sdk/v3/backend"
+import "github.com/cloudquery/plugin-sdk/v4/state"
 
 type Backend interface {
-	backend.Backend
+	state.Client
 }

client/client.go

@@ -1,21 +1,18 @@
 package client
 
 import (
-	"context"
 	"fmt"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"time"
 
-	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v4/state"
 	"github.com/coinpaprika/coinpaprika-api-go-client/v2/coinpaprika"
 	"github.com/rs/zerolog"
 )
 
 type Client struct {
-	Logger            zerolog.Logger
 	CoinpaprikaClient CoinpaprikaServices
-	Backend           Backend
+	Backend           state.Client
 	StartDate         time.Time
 	EndDate           time.Time
 	Interval          string
@@ -26,13 +23,7 @@ func (c *Client) ID() string {
 	return "coinpaprika"
 }
 
-func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
-	var pluginSpec Spec
-
-	if err := s.UnmarshalSpec(&pluginSpec); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
-	}
-
+func New(logger zerolog.Logger, pluginSpec Spec, backend state.Client) (schema.ClientMeta, error) {
 	var cOpts []coinpaprika.ClientOptions
 	if pluginSpec.AccessToken != "" {
 		cOpts = append(cOpts, coinpaprika.WithAPIKey(pluginSpec.AccessToken))
@@ -62,13 +53,12 @@ func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source
 	cc := coinpaprika.NewClient(NewHttpClient(logger, pluginSpec.ApiDebug, rateNumber, rateDuration), cOpts...)
 
 	return &Client{
-		Logger: logger,
 		CoinpaprikaClient: CoinpaprikaServices{
 			Tickers:   &cc.Tickers,
 			Coins:     &cc.Coins,
 			Exchanges: &cc.Exchanges,
 		},
-		Backend:   opts.Backend,
+		Backend:   backend,
 		StartDate: startDate,
 		EndDate:   endDate,
 		Interval:  pluginSpec.Interval,

client/testing.go

@@ -2,59 +2,52 @@ package client
 
 import (
 	"context"
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/scheduler"
+	"github.com/cloudquery/plugin-sdk/v4/state"
+	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"os"
 	"testing"
 	"time"
 
-	"github.com/cloudquery/plugin-pb-go/specs"
-	"github.com/cloudquery/plugin-sdk/v3/backend"
-	"github.com/cloudquery/plugin-sdk/v3/plugins/source"
-	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/golang/mock/gomock"
 	"github.com/rs/zerolog"
 )
 
 type TestOptions struct {
-	Backend   backend.Backend
+	Backend   state.Client
 	StartTime time.Time
 	EndTime   time.Time
 	Interval  string
 	Tickers   []string
 }
 
 func MockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.T, *gomock.Controller) CoinpaprikaServices, opts TestOptions) {
-	version := "vDev"
 	table.IgnoreInTests = false
 	t.Helper()
 	ctrl := gomock.NewController(t)
 	l := zerolog.New(zerolog.NewTestWriter(t)).Output(
 		zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro},
 	).Level(zerolog.DebugLevel).With().Timestamp().Logger()
 
-	newTestExecutionClient := func(ctx context.Context, logger zerolog.Logger, spec specs.Source, _ source.Options) (schema.ClientMeta, error) {
-		return &Client{
-			Logger:            l,
-			CoinpaprikaClient: builder(t, ctrl),
-			Backend:           opts.Backend,
-			StartDate:         opts.StartTime,
-			EndDate:           opts.EndTime,
-			Interval:          opts.Interval,
-			Tickers:           opts.Tickers,
-		}, nil
+	client := &Client{
+		CoinpaprikaClient: builder(t, ctrl),
+		Backend:           opts.Backend,
+		StartDate:         opts.StartTime,
+		EndDate:           opts.EndTime,
+		Interval:          opts.Interval,
+		Tickers:           opts.Tickers,
 	}
-	p := source.NewPlugin(
-		table.Name,
-		version,
-		[]*schema.Table{
-			table,
-		},
-		newTestExecutionClient)
-	p.SetLogger(l)
-	source.TestPluginSync(t, p, specs.Source{
-		Name:         "dev",
-		Path:         "cloudquery/dev",
-		Version:      version,
-		Tables:       []string{table.Name},
-		Destinations: []string{"mock-destination"},
-	})
+
+	tables := schema.Tables{table}
+	if err := transformers.TransformTables(tables); err != nil {
+		t.Fatal(err)
+	}
+	sched := scheduler.NewScheduler(scheduler.WithLogger(l))
+	messages, err := sched.SyncAll(context.Background(), client, tables)
+	if err != nil {
+		t.Fatalf("failed to sync: %v", err)
+	}
+	plugin.ValidateNoEmptyColumns(t, tables, messages)
 }

docs/tables/README.md

@@ -1,4 +1,4 @@
-# Source Plugin: coinpaprika-coinpaprika
+# Source Plugin: coinpaprika
 
 ## Tables
 

docs/tables/coinpaprika_coins.md

@@ -15,8 +15,6 @@ The following tables depend on coinpaprika_coins:
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
 |id (PK)|`utf8`|

docs/tables/coinpaprika_exchanges.md

@@ -2,17 +2,15 @@
 
 This table shows data for Coinpaprika Exchanges.
 
-The primary key for this table is **_cq_id**.
+The primary key for this table is **id**.
 
 ## Columns
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
-|_cq_id (PK)|`uuid`|
+|_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
-|id|`utf8`|
+|id (PK)|`utf8`|
 |name|`utf8`|
 |message|`utf8`|
 |description|`utf8`|

docs/tables/coinpaprika_tickers.md

@@ -4,7 +4,7 @@ This table shows data for Coinpaprika Tickers.
 
 https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById
 
-The composite primary key for this table is (**id**, **timestamp**).
+The composite primary key for this table is (**coin_id**, **timestamp**).
 It supports incremental syncs based on the **timestamp** column.
 ## Relations
 
@@ -14,12 +14,10 @@ This table depends on [coinpaprika_coins](coinpaprika_coins.md).
 
 | Name          | Type          |
 | ------------- | ------------- |
-|_cq_source_name|`utf8`|
-|_cq_sync_time|`timestamp[us, tz=UTC]`|
 |_cq_id|`uuid`|
 |_cq_parent_id|`uuid`|
-|id (PK)|`utf8`|
-|timestamp (PK) (Incremental Key)|`utf8`|
+|coin_id (PK)|`utf8`|
+|timestamp (PK) (Incremental Key)|`timestamp[us, tz=UTC]`|
 |price|`float64`|
 |volume_24h|`float64`|
 |market_cap|`float64`|