Merge branch 'master' of github.com:rudderlabs/rudder-server into cho…

…re.uploads-state
rudderlabs · Oct 31, 2023 · 9f47bba · 9f47bba
2 parents 4cd6902 + 6d2db45
commit 9f47bba
Show file tree

Hide file tree

Showing 100 changed files with 4,495 additions and 2,005 deletions.
diff --git a/.github/tools/matrixchecker/main.go b/.github/tools/matrixchecker/main.go
@@ -4,8 +4,8 @@ import (
  "log"
  "os"
  "path/filepath"
+ "slices"
 
- "golang.org/x/exp/slices"
  "gopkg.in/yaml.v2"
 )
 
@@ -20,6 +20,7 @@ var IgnorePackages = []string{
  "warehouse/integrations/testdata",
  "warehouse/integrations/config",
  "warehouse/integrations/types",
+ "warehouse/integrations/tunnelling",
 }
 
 func main() {

diff --git a/archiver/archiver_isolation_test.go b/archiver/archiver_isolation_test.go
@@ -14,13 +14,16 @@ import (
  "testing"
  "time"
 
+ "github.com/rudderlabs/rudder-go-kit/bytesize"
+ "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres"
  "github.com/rudderlabs/rudder-server/testhelper/destination"
 
+ "golang.org/x/sync/errgroup"
+
  "github.com/google/uuid"
  "github.com/ory/dockertest/v3"
  "github.com/stretchr/testify/require"
  "github.com/tidwall/gjson"
- "golang.org/x/sync/errgroup"
 
  "github.com/rudderlabs/rudder-go-kit/config"
  "github.com/rudderlabs/rudder-go-kit/filemanager"
@@ -127,7 +130,7 @@ func ArchivalScenario(
  cleanup := &testhelper.Cleanup{}
  defer cleanup.Run()
 
- postgresContainer, err := resource.SetupPostgres(pool, cleanup)
+ postgresContainer, err := resource.SetupPostgres(pool, cleanup, postgres.WithShmSize(256*bytesize.MB))
  require.NoError(t, err, "failed to setup postgres container")
 
  minioResource, err := resource.SetupMinio(pool, cleanup)

diff --git a/archiver/archiver_test.go b/archiver/archiver_test.go
@@ -17,12 +17,12 @@ import (
  "github.com/stretchr/testify/require"
  "github.com/tidwall/gjson"
 
- "github.com/rudderlabs/rudder-go-kit/config"
- "github.com/rudderlabs/rudder-go-kit/stats"
-
  "github.com/rudderlabs/rudder-go-kit/bytesize"
+ "github.com/rudderlabs/rudder-go-kit/config"
  "github.com/rudderlabs/rudder-go-kit/filemanager"
+ "github.com/rudderlabs/rudder-go-kit/stats"
  "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource"
+ "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres"
  trand "github.com/rudderlabs/rudder-go-kit/testhelper/rand"
  backendconfig "github.com/rudderlabs/rudder-server/backend-config"
  "github.com/rudderlabs/rudder-server/jobsdb"
@@ -47,7 +47,7 @@ func TestJobsArchival(t *testing.T) {
  pool, err := dockertest.NewPool("")
  require.NoError(t, err, "Failed to create docker pool")
 
- postgresResource, err := resource.SetupPostgres(pool, t)
+ postgresResource, err := resource.SetupPostgres(pool, t, postgres.WithShmSize(256*bytesize.MB))
  require.NoError(t, err, "failed to setup postgres resource")
  c := config.New()
  c.Set("DB.name", postgresResource.Database)
@@ -205,7 +205,7 @@ func readGzipJobFile(filename string) ([]*jobsdb.JobT, error) {
  if err != nil {
  return []*jobsdb.JobT{}, err
  }
- defer gz.Close()
+ defer func() { _ = gz.Close() }()
 
  sc := bufio.NewScanner(gz)
  // default scanner buffer maxCapacity is 64K
@@ -255,24 +255,24 @@ type jdWrapper struct {
  queries *int32
 }
 
-func (jd jdWrapper) GetDistinctParameterValues(ctx context.Context, parameterName string) ([]string, error) {
+func (jd jdWrapper) GetDistinctParameterValues(context.Context, string) ([]string, error) {
  atomic.AddInt32(jd.queries, 1)
  return []string{}, nil
 }
 
 func (jd jdWrapper) GetUnprocessed(
- ctx context.Context,
- params jobsdb.GetQueryParams,
+ context.Context,
+ jobsdb.GetQueryParams,
 ) (jobsdb.JobsResult, error) {
  atomic.AddInt32(jd.queries, 1)
  return jobsdb.JobsResult{}, nil
 }
 
 func (jd jdWrapper) UpdateJobStatus(
- ctx context.Context,
- statusList []*jobsdb.JobStatusT,
- customValFilters []string,
- parameterFilters []jobsdb.ParameterFilterT,
+ context.Context,
+ []*jobsdb.JobStatusT,
+ []string,
+ []jobsdb.ParameterFilterT,
 ) error {
  atomic.AddInt32(jd.queries, 1)
  return nil

diff --git a/controlplane/controlplane.go b/controlplane/controlplane.go
@@ -38,7 +38,7 @@ func (cm *ConnectionManager) establishConnection() (*ConnHandler, error) {
  return nil, err
  }
 
- grpcServer := grpc.NewServer()
+ grpcServer := grpc.NewServer(cm.Options...)
  service := &authService{authInfo: cm.AuthInfo}
  proto.RegisterDPAuthServiceServer(grpcServer, service)
  cn := &ConnHandler{

diff --git a/controlplane/manager.go b/controlplane/manager.go
@@ -18,6 +18,7 @@ type ConnectionManager struct {
  active bool
  url string
  connHandler *ConnHandler
+ Options []grpc.ServerOption
 }
 
 type LoggerI interface {