Skip to content

Commit

Permalink
fixup! chore(deps): bump cloud.google.com/go/bigquery from 1.59.1 to …
Browse files Browse the repository at this point in the history
…1.61.0
  • Loading branch information
atzoum committed Apr 29, 2024
1 parent 4788717 commit 86ec4ed
Show file tree
Hide file tree
Showing 5 changed files with 5 additions and 67 deletions.
3 changes: 0 additions & 3 deletions sqlconnect/internal/bigquery/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,6 @@ func NewDB(configJSON json.RawMessage) (*DB, error) {

db := sql.OpenDB(driver.NewConnector(
config.ProjectID,
driver.Config{
JobRateLimitExceededRetryEnabled: true,
},
option.WithCredentialsJSON([]byte(config.CredentialsJSON))),
)

Expand Down
29 changes: 0 additions & 29 deletions sqlconnect/internal/bigquery/driver/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,12 @@ import (
"database/sql/driver"
"errors"
"fmt"
"strings"
"time"

"cloud.google.com/go/bigquery"
"github.com/cenkalti/backoff/v4"
"google.golang.org/api/iterator"
)

type bigQueryConnection struct {
config Config
ctx context.Context
client *bigquery.Client
closed bool
Expand Down Expand Up @@ -80,28 +76,3 @@ func (bigQueryConnection) CheckNamedValue(*driver.NamedValue) error {
func (connection *bigQueryConnection) BigqueryClient() *bigquery.Client {
return connection.client
}

// readWithBackoff will retry the read operation if the error is [jobRateLimitExceeded] and the config is set to retry rate limit errors
//
// TODO: this should no longer be needed once this fix is released:
// https://github.com/googleapis/google-cloud-go/pull/9726
func (connection *bigQueryConnection) readWithBackoff(ctx context.Context, query *bigquery.Query) (it *bigquery.RowIterator, err error) {
if !connection.config.JobRateLimitExceededRetryEnabled {
return query.Read(ctx)
}
// mimicking google's own retry backoff settings
// https://github.com/googleapis/google-cloud-go/blob/b2e704d9d287445304d2b6030b6e35a4eb8be80a/bigquery/bigquery.go#L236
retry := backoff.WithContext(backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(32*time.Second),
backoff.WithMultiplier(2),
), ctx)
_ = backoff.Retry(func() error {
it, err = query.Read(ctx)
if err != nil && (strings.Contains(err.Error(), "jobRateLimitExceeded")) {
return err
}
return nil
}, retry)
return it, err
}
9 changes: 1 addition & 8 deletions sqlconnect/internal/bigquery/driver/connector.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,15 @@ import (
"google.golang.org/api/option"
)

type Config struct {
JobRateLimitExceededRetryEnabled bool // Enable jobRateLimitExceeded retries: default false
}

func NewConnector(projectID string, config Config, opts ...option.ClientOption) driver.Connector {
func NewConnector(projectID string, opts ...option.ClientOption) driver.Connector {
return &bigQueryConnector{
projectID: projectID,
config: config,
opts: opts,
}
}

type bigQueryConnector struct {
projectID string
config Config
opts []option.ClientOption
}

Expand All @@ -33,7 +27,6 @@ func (c *bigQueryConnector) Connect(ctx context.Context) (driver.Conn, error) {
}

return &bigQueryConnection{
config: c.config,
ctx: ctx,
client: client,
}, nil
Expand Down
27 changes: 2 additions & 25 deletions sqlconnect/internal/bigquery/driver/driver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestBigqueryDriver(t *testing.T) {
require.NoError(t, json.Unmarshal([]byte(configJSON), &c))

t.Run("OpenDB", func(t *testing.T) {
db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
db := sql.OpenDB(driver.NewConnector(c.ProjectID, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
t.Cleanup(func() {
require.NoError(t, db.Close(), "it should be able to close the database connection")
})
Expand Down Expand Up @@ -225,32 +225,9 @@ func TestBigqueryDriver(t *testing.T) {
})
})

// TODO: this test should start failing once this fix is released:
// https://github.com/googleapis/google-cloud-go/pull/9726
t.Run("jobRateLimitExceeded", func(t *testing.T) {
t.Run("selecting from information schema from multiple go-routines should lead to a jobRateLimitExceeded error", func(t *testing.T) {
db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
t.Cleanup(func() {
require.NoError(t, db.Close(), "it should be able to close the database connection")
})
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)

g, ctx := errgroup.WithContext(ctx)
g.SetLimit(10)
for i := 0; i < 200; i++ {
g.Go(func() error {
_, err := db.ExecContext(ctx, "SELECT * FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'invalid'")
return err
})
}
err := g.Wait()
require.Error(t, err, "it should return an error")
require.Contains(t, err.Error(), "jobRateLimitExceeded", "it should return a jobRateLimitExceeded error")
})

t.Run("selecting from information schema from multiple go-routines and having rate limit retries enabled, shoudln't lead to a jobRateLimitExceeded error", func(t *testing.T) {
db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{JobRateLimitExceededRetryEnabled: true}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
db := sql.OpenDB(driver.NewConnector(c.ProjectID, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
t.Cleanup(func() {
require.NoError(t, db.Close(), "it should be able to close the database connection")
})
Expand Down
4 changes: 2 additions & 2 deletions sqlconnect/internal/bigquery/driver/statement.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (statement *bigQueryStatement) ExecContext(ctx context.Context, args []driv
return nil, err
}

rowIterator, err := statement.connection.readWithBackoff(ctx, query)
rowIterator, err := query.Read(ctx)
if err != nil {
return nil, err
}
Expand All @@ -55,7 +55,7 @@ func (statement *bigQueryStatement) QueryContext(ctx context.Context, args []dri
return nil, err
}

rowIterator, err := statement.connection.readWithBackoff(ctx, query)
rowIterator, err := query.Read(ctx)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 86ec4ed

Please sign in to comment.