chore: bump bigquery to v1.61.0 (#78)

# Description

bump bigquery to 1.61.0

## Security

- [x] The code changed/added as part of this pull request won't create
any security issues with how the software is being used.

go.mod

@@ -6,21 +6,20 @@ toolchain go1.22.2
 
 require (
  cloud.google.com/go v0.113.0
- cloud.google.com/go/bigquery v1.60.0
+ cloud.google.com/go/bigquery v1.61.0
  github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
  github.com/aws/aws-sdk-go-v2 v1.27.0
  github.com/aws/aws-sdk-go-v2/config v1.27.11
  github.com/aws/aws-sdk-go-v2/credentials v1.17.15
  github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.25.7
- github.com/cenkalti/backoff/v4 v4.3.0
  github.com/databricks/databricks-sql-go v1.5.5
  github.com/dlclark/regexp2 v1.11.0
  github.com/gliderlabs/ssh v0.3.7
  github.com/go-sql-driver/mysql v1.8.1
  github.com/google/uuid v1.6.0
  github.com/lib/pq v1.10.9
  github.com/ory/dockertest/v3 v3.10.0
- github.com/rudderlabs/rudder-go-kit v0.30.0
+ github.com/rudderlabs/rudder-go-kit v0.31.0
  github.com/rudderlabs/sql-tunnels v0.1.6
  github.com/samber/lo v1.39.0
  github.com/sirupsen/logrus v1.9.3
@@ -30,7 +29,6 @@ require (
  github.com/tidwall/sjson v1.2.5
  github.com/trinodb/trino-go-client v0.315.0
  golang.org/x/crypto v0.23.0
- golang.org/x/sync v0.7.0
  google.golang.org/api v0.181.0
 )
 
@@ -51,7 +49,6 @@ require (
  github.com/Microsoft/go-winio v0.6.1 // indirect
  github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
  github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
- github.com/apache/arrow/go/v14 v14.0.2 // indirect
  github.com/apache/arrow/go/v15 v15.0.2 // indirect
  github.com/apache/arrow/go/v16 v16.0.0 // indirect
  github.com/apache/thrift v0.19.0 // indirect
@@ -71,6 +68,7 @@ require (
  github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.2 // indirect
  github.com/aws/aws-sdk-go-v2/service/sts v1.28.9 // indirect
  github.com/aws/smithy-go v1.20.2 // indirect
+ github.com/cenkalti/backoff/v4 v4.3.0 // indirect
  github.com/containerd/continuity v0.4.3 // indirect
  github.com/coreos/go-oidc/v3 v3.5.0 // indirect
  github.com/danieljoos/wincred v1.1.2 // indirect
@@ -131,13 +129,14 @@ require (
  go.opencensus.io v0.24.0 // indirect
  go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
- go.opentelemetry.io/otel v1.25.0 // indirect
- go.opentelemetry.io/otel/metric v1.25.0 // indirect
- go.opentelemetry.io/otel/trace v1.25.0 // indirect
+ go.opentelemetry.io/otel v1.26.0 // indirect
+ go.opentelemetry.io/otel/metric v1.26.0 // indirect
+ go.opentelemetry.io/otel/trace v1.26.0 // indirect
  golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
  golang.org/x/mod v0.17.0 // indirect
  golang.org/x/net v0.25.0 // indirect
  golang.org/x/oauth2 v0.20.0 // indirect
+ golang.org/x/sync v0.7.0 // indirect
  golang.org/x/sys v0.20.0 // indirect
  golang.org/x/term v0.20.0 // indirect
  golang.org/x/text v0.15.0 // indirect

go.sum

@@ -5,8 +5,8 @@ cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg=
 cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro=
 cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
 cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
-cloud.google.com/go/bigquery v1.60.0 h1:kA96WfgvCbkqfLnr7xI5uEfJ4h4FrnkdEb0yty0KSZo=
-cloud.google.com/go/bigquery v1.60.0/go.mod h1:Clwk2OeC0ZU5G5LDg7mo+h8U7KlAa5v06z5rptKdM3g=
+cloud.google.com/go/bigquery v1.61.0 h1:w2Goy9n6gh91LVi6B2Sc+HpBl8WbWhIyzdvVvrAuEIw=
+cloud.google.com/go/bigquery v1.61.0/go.mod h1:PjZUje0IocbuTOdq4DBOJLNYB0WF3pAKBHzAYyxCwFo=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
 cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
@@ -51,8 +51,6 @@ github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:3YVZUqkoev4mL+aCwVO
 github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:ymXt5bw5uSNu4jveerFxE0vNYxF8ncqbptntMaFMg3k=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
-github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw=
-github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
 github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=
 github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=
 github.com/apache/arrow/go/v16 v16.0.0 h1:qRLbJRPj4zaseZrjbDHa7mUoZDDIU+4pu+mE2Lucs5g=
@@ -295,8 +293,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
 github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
-github.com/rudderlabs/rudder-go-kit v0.30.0 h1:JV5NlnvAboh//2mo2sV3PDXuHPX5Y6CtjZXibMc1wWM=
-github.com/rudderlabs/rudder-go-kit v0.30.0/go.mod h1:Io1s43ApHrIMUUrz0Z2j+zb9p8r49NKT5YUzMIh62Q8=
+github.com/rudderlabs/rudder-go-kit v0.31.0 h1:7bHJMogiMvkrlGAIy/FrDfR8eUIuJBfGbZcnolkmvW4=
+github.com/rudderlabs/rudder-go-kit v0.31.0/go.mod h1:EJSKYDnbUjJzBP8ZhniW6GkfxfK2nBTaxiL5KBYvo08=
 github.com/rudderlabs/sql-tunnels v0.1.6 h1:v2KA2cq8ZV5LXRJQpqigq1Q4V64oDL+XlfckW/0K2/4=
 github.com/rudderlabs/sql-tunnels v0.1.6/go.mod h1:Jew/XwojzFTK4KTDj/wvChI7iZCgAKYyKjFK1nnHlNM=
 github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
@@ -353,14 +351,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
-go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k=
-go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg=
-go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA=
-go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s=
-go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo=
-go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw=
-go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM=
-go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I=
+go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
+go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
+go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
+go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
+go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8=
+go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs=
+go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
+go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

sqlconnect/internal/bigquery/db.go

@@ -30,9 +30,6 @@ func NewDB(configJSON json.RawMessage) (*DB, error) {
 
  db := sql.OpenDB(driver.NewConnector(
  config.ProjectID,
- driver.Config{
- JobRateLimitExceededRetryEnabled: true,
- },
  option.WithCredentialsJSON([]byte(config.CredentialsJSON))),
  )
 

sqlconnect/internal/bigquery/driver/connection.go

@@ -5,16 +5,12 @@ import (
  "database/sql/driver"
  "errors"
  "fmt"
- "strings"
- "time"
 
  "cloud.google.com/go/bigquery"
- "github.com/cenkalti/backoff/v4"
  "google.golang.org/api/iterator"
 )
 
 type bigQueryConnection struct {
- config Config
  ctx context.Context
  client *bigquery.Client
  closed bool
@@ -80,28 +76,3 @@ func (bigQueryConnection) CheckNamedValue(*driver.NamedValue) error {
 func (connection *bigQueryConnection) BigqueryClient() *bigquery.Client {
  return connection.client
 }
-
-// readWithBackoff will retry the read operation if the error is [jobRateLimitExceeded] and the config is set to retry rate limit errors
-//
-// TODO: this should no longer be needed once this fix is released:
-// https://github.com/googleapis/google-cloud-go/pull/9726
-func (connection *bigQueryConnection) readWithBackoff(ctx context.Context, query *bigquery.Query) (it *bigquery.RowIterator, err error) {
- if !connection.config.JobRateLimitExceededRetryEnabled {
- return query.Read(ctx)
- }
- // mimicking google's own retry backoff settings
- // https://github.com/googleapis/google-cloud-go/blob/b2e704d9d287445304d2b6030b6e35a4eb8be80a/bigquery/bigquery.go#L236
- retry := backoff.WithContext(backoff.NewExponentialBackOff(
- backoff.WithInitialInterval(1*time.Second),
- backoff.WithMaxInterval(32*time.Second),
- backoff.WithMultiplier(2),
- ), ctx)
- _ = backoff.Retry(func() error {
- it, err = query.Read(ctx)
- if err != nil && (strings.Contains(err.Error(), "jobRateLimitExceeded")) {
- return err
- }
- return nil
- }, retry)
- return it, err
-}

sqlconnect/internal/bigquery/driver/connector.go

@@ -8,21 +8,15 @@ import (
  "google.golang.org/api/option"
 )
 
-type Config struct {
- JobRateLimitExceededRetryEnabled bool // Enable jobRateLimitExceeded retries: default false
-}
-
-func NewConnector(projectID string, config Config, opts ...option.ClientOption) driver.Connector {
+func NewConnector(projectID string, opts ...option.ClientOption) driver.Connector {
  return &bigQueryConnector{
  projectID: projectID,
- config: config,
  opts: opts,
  }
 }
 
 type bigQueryConnector struct {
  projectID string
- config Config
  opts []option.ClientOption
 }
 
@@ -33,7 +27,6 @@ func (c *bigQueryConnector) Connect(ctx context.Context) (driver.Conn, error) {
  }
 
  return &bigQueryConnection{
- config: c.config,
  ctx: ctx,
  client: client,
  }, nil

sqlconnect/internal/bigquery/driver/driver_test.go

@@ -12,7 +12,6 @@ import (
  "time"
 
  "github.com/stretchr/testify/require"
- "golang.org/x/sync/errgroup"
  "google.golang.org/api/option"
 
  "github.com/rudderlabs/rudder-go-kit/testhelper/rand"
@@ -34,7 +33,7 @@ func TestBigqueryDriver(t *testing.T) {
  require.NoError(t, json.Unmarshal([]byte(configJSON), &c))
 
  t.Run("OpenDB", func(t *testing.T) {
- db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
+ db := sql.OpenDB(driver.NewConnector(c.ProjectID, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
  t.Cleanup(func() {
  require.NoError(t, db.Close(), "it should be able to close the database connection")
  })
@@ -224,51 +223,6 @@ func TestBigqueryDriver(t *testing.T) {
  require.NoError(t, rows.Err())
  })
  })
-
- // TODO: this test should start failing once this fix is released:
- // https://github.com/googleapis/google-cloud-go/pull/9726
- t.Run("jobRateLimitExceeded", func(t *testing.T) {
- t.Run("selecting from information schema from multiple go-routines should lead to a jobRateLimitExceeded error", func(t *testing.T) {
- db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
- t.Cleanup(func() {
- require.NoError(t, db.Close(), "it should be able to close the database connection")
- })
- ctx, cancel := context.WithCancel(context.Background())
- t.Cleanup(cancel)
-
- g, ctx := errgroup.WithContext(ctx)
- g.SetLimit(10)
- for i := 0; i < 200; i++ {
- g.Go(func() error {
- _, err := db.ExecContext(ctx, "SELECT * FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'invalid'")
- return err
- })
- }
- err := g.Wait()
- require.Error(t, err, "it should return an error")
- require.Contains(t, err.Error(), "jobRateLimitExceeded", "it should return a jobRateLimitExceeded error")
- })
-
- t.Run("selecting from information schema from multiple go-routines and having rate limit retries enabled, shoudln't lead to a jobRateLimitExceeded error", func(t *testing.T) {
- db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{JobRateLimitExceededRetryEnabled: true}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
- t.Cleanup(func() {
- require.NoError(t, db.Close(), "it should be able to close the database connection")
- })
- ctx, cancel := context.WithCancel(context.Background())
- t.Cleanup(cancel)
-
- g, ctx := errgroup.WithContext(ctx)
- g.SetLimit(10)
- for i := 0; i < 50; i++ {
- g.Go(func() error {
- _, err := db.ExecContext(ctx, "SELECT * FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'invalid'")
- return err
- })
- }
- err := g.Wait()
- require.NoError(t, err, "it shouldn't return an error")
- })
- })
 }
 
 type config struct {

sqlconnect/internal/bigquery/driver/statement.go

@@ -41,7 +41,7 @@ func (statement *bigQueryStatement) ExecContext(ctx context.Context, args []driv
  return nil, err
  }
 
- rowIterator, err := statement.connection.readWithBackoff(ctx, query)
+ rowIterator, err := query.Read(ctx)
  if err != nil {
  return nil, err
  }
@@ -55,7 +55,7 @@ func (statement *bigQueryStatement) QueryContext(ctx context.Context, args []dri
  return nil, err
  }
 
- rowIterator, err := statement.connection.readWithBackoff(ctx, query)
+ rowIterator, err := query.Read(ctx)
  if err != nil {
  return nil, err
  }