kvtenant: expose available regions to tenants

docs/generated/http/full.md

@@ -142,6 +142,73 @@ SystemInfo contains information about the host system.
 
 
 
+## Regions
+
+
+
+RegionsRequest retrieves all available regions.
+
+Support status: [reserved](#support-status)
+
+#### Request Parameters
+
+
+
+
+RegionsRequest requests all available regions.
+
+
+
+
+
+
+
+
+#### Response Parameters
+
+
+
+
+RegionsResponse describes the available regions.
+
+
+| Field | Type | Label | Description | Support status |
+| ----- | ---- | ----- | ----------- | -------------- |
+| regions | [RegionsResponse.RegionsEntry](#cockroach.server.serverpb.RegionsResponse-cockroach.server.serverpb.RegionsResponse.RegionsEntry) | repeated |  | [reserved](#support-status) |
+
+
+
+
+
+
+<a name="cockroach.server.serverpb.RegionsResponse-cockroach.server.serverpb.RegionsResponse.RegionsEntry"></a>
+#### RegionsResponse.RegionsEntry
+
+
+
+| Field | Type | Label | Description | Support status |
+| ----- | ---- | ----- | ----------- | -------------- |
+| key | [string](#cockroach.server.serverpb.RegionsResponse-string) |  |  |  |
+| value | [RegionsResponse.Region](#cockroach.server.serverpb.RegionsResponse-cockroach.server.serverpb.RegionsResponse.Region) |  |  |  |
+
+
+
+
+
+<a name="cockroach.server.serverpb.RegionsResponse-cockroach.server.serverpb.RegionsResponse.Region"></a>
+#### RegionsResponse.Region
+
+
+
+| Field | Type | Label | Description | Support status |
+| ----- | ---- | ----- | ----------- | -------------- |
+| zones | [string](#cockroach.server.serverpb.RegionsResponse-string) | repeated |  | [reserved](#support-status) |
+
+
+
+
+
+
 ## Nodes
 
 `GET /_status/nodes`

pkg/ccl/backupccl/testdata/backup-restore/multiregion

@@ -8,7 +8,7 @@ INSERT INTO d.t VALUES (1), (2), (3);
 ----
 
 query-sql
-SELECT region FROM [SHOW REGIONS FROM DATABASE d];
+SELECT region FROM [SHOW REGIONS FROM DATABASE d] ORDER BY 1;
 ----
 eu-central-1
 us-east-1

pkg/ccl/kvccl/kvtenantccl/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//pkg/kv/kvclient/rangecache",
         "//pkg/roachpb",
         "//pkg/rpc",
+        "//pkg/server/serverpb",
         "//pkg/util/contextutil",
         "//pkg/util/grpcutil",
         "//pkg/util/log",

pkg/ccl/kvccl/kvtenantccl/connector.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/util/contextutil"
 	"github.com/cockroachdb/cockroach/pkg/util/grpcutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -64,13 +65,19 @@ type Connector struct {
 
 	mu struct {
 		syncutil.RWMutex
-		client               roachpb.InternalClient
+		client               *client
 		nodeDescs            map[roachpb.NodeID]*roachpb.NodeDescriptor
 		systemConfig         *config.SystemConfig
 		systemConfigChannels []chan<- struct{}
 	}
 }
 
+// client represents an RPC client that proxies to a KV instance.
+type client struct {
+	roachpb.InternalClient
+	serverpb.StatusClient
+}
+
 // Connector is capable of providing information on each of the KV nodes in the
 // cluster in the form of NodeDescriptors. This obviates the need for SQL-only
 // tenant processes to join the cluster-wide gossip network.
@@ -90,6 +97,11 @@ var _ rangecache.RangeDescriptorDB = (*Connector)(nil)
 // network.
 var _ config.SystemConfigProvider = (*Connector)(nil)
 
+// Connector is capable of find the region of every node in the cluster.
+// This is necessary for region validation for zone configurations and
+// multi-region primitives.
+var _ serverpb.RegionsServer = (*Connector)(nil)
+
 // NewConnector creates a new Connector.
 // NOTE: Calling Start will set cfg.RPCContext.ClusterID.
 func NewConnector(cfg kvtenant.ConnectorConfig, addrs []string) *Connector {
@@ -346,6 +358,21 @@ func (c *Connector) RangeLookup(
 	return nil, nil, ctx.Err()
 }
 
+// Regions implements the serverpb.RegionsServer interface.
+func (c *Connector) Regions(
+	ctx context.Context, req *serverpb.RegionsRequest,
+) (*serverpb.RegionsResponse, error) {
+	ctx = c.AnnotateCtx(ctx)
+	for ctx.Err() == nil {
+		client, err := c.getClient(ctx)
+		if err != nil {
+			continue
+		}
+		return client.Regions(ctx, req)
+	}
+	return nil, ctx.Err()
+}
+
 // FirstRange implements the kvcoord.RangeDescriptorDB interface.
 func (c *Connector) FirstRange() (*roachpb.RangeDescriptor, error) {
 	return nil, status.Error(codes.Unauthenticated, "kvtenant.Proxy does not have access to FirstRange")
@@ -355,7 +382,7 @@ func (c *Connector) FirstRange() (*roachpb.RangeDescriptor, error) {
 // not, the method attempts to dial one of the configured addresses. The method
 // blocks until either a connection is successfully established or the provided
 // context is canceled.
-func (c *Connector) getClient(ctx context.Context) (roachpb.InternalClient, error) {
+func (c *Connector) getClient(ctx context.Context) (*client, error) {
 	c.mu.RLock()
 	if client := c.mu.client; client != nil {
 		c.mu.RUnlock()
@@ -365,7 +392,7 @@ func (c *Connector) getClient(ctx context.Context) (roachpb.InternalClient, erro
 		dialCtx := c.AnnotateCtx(context.Background())
 		dialCtx, cancel := c.rpcContext.Stopper.WithCancelOnQuiesce(dialCtx)
 		defer cancel()
-		var client roachpb.InternalClient
+		var client *client
 		err := c.rpcContext.Stopper.RunTaskWithErr(dialCtx, "kvtenant.Connector: dial",
 			func(ctx context.Context) error {
 				var err error
@@ -387,15 +414,15 @@ func (c *Connector) getClient(ctx context.Context) (roachpb.InternalClient, erro
 		if res.Err != nil {
 			return nil, res.Err
 		}
-		return res.Val.(roachpb.InternalClient), nil
+		return res.Val.(*client), nil
 	case <-ctx.Done():
 		return nil, ctx.Err()
 	}
 }
 
 // dialAddrs attempts to dial each of the configured addresses in a retry loop.
 // The method will only return a non-nil error on context cancellation.
-func (c *Connector) dialAddrs(ctx context.Context) (roachpb.InternalClient, error) {
+func (c *Connector) dialAddrs(ctx context.Context) (*client, error) {
 	for r := retry.StartWithCtx(ctx, c.rpcRetryOptions); r.Next(); {
 		// Try each address on each retry iteration.
 		randStart := rand.Intn(len(c.addrs))
@@ -406,7 +433,10 @@ func (c *Connector) dialAddrs(ctx context.Context) (roachpb.InternalClient, erro
 				log.Warningf(ctx, "error dialing tenant KV address %s: %v", addr, err)
 				continue
 			}
-			return roachpb.NewInternalClient(conn), nil
+			return &client{
+				InternalClient: roachpb.NewInternalClient(conn),
+				StatusClient:   serverpb.NewStatusClient(conn),
+			}, nil
 		}
 	}
 	return nil, ctx.Err()

pkg/ccl/logictestccl/testdata/logic_test/multi_region_tenant

@@ -0,0 +1,14 @@
+# LogicTest: multiregion-9node-3region-3azs-tenant
+
+query TT
+SHOW REGIONS FROM CLUSTER
+----
+ap-southeast-2  {ap-az1,ap-az2,ap-az3}
+ca-central-1    {ca-az1,ca-az2,ca-az3}
+us-east-1       {us-az1,us-az2,us-az3}
+
+statement error region "bad-region" does not exist
+CREATE DATABASE db PRIMARY REGION "bad-region"
+
+statement error operation is unsupported in multi-tenancy mode
+CREATE DATABASE db PRIMARY REGION "us-east-1"

pkg/ccl/multiregionccl/region_test.go

@@ -9,6 +9,7 @@
 package multiregionccl_test
 
 import (
+	"sort"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -183,21 +184,9 @@ CREATE TABLE db.rbr () LOCALITY REGIONAL BY ROW`)
 				dbRegions = append(dbRegions, region)
 			}
 
-			if len(dbRegions) != len(tc.expectedRegions) {
-				t.Fatalf("unexpected number of regions, expected: %v found %v",
-					tc.expectedRegions,
-					dbRegions,
-				)
-			}
-
-			for i, expectedRegion := range tc.expectedRegions {
-				if expectedRegion != dbRegions[i] {
-					t.Fatalf("unexpected regions, expected: %v found %v",
-						tc.expectedRegions,
-						dbRegions,
-					)
-				}
-			}
+			sort.Strings(tc.expectedRegions)
+			sort.Strings(dbRegions)
+			require.Equal(t, tc.expectedRegions, dbRegions)
 		})
 	}
 }

pkg/cli/testdata/zip/partial1

@@ -24,6 +24,7 @@ debug zip --concurrency=1 --cpu-profile-duration=0s /dev/null
 [cluster] retrieving SQL data for crdb_internal.kv_node_liveness... writing output: debug/crdb_internal.kv_node_liveness.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_node_status... writing output: debug/crdb_internal.kv_node_status.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_store_status... writing output: debug/crdb_internal.kv_store_status.txt... done
+[cluster] retrieving SQL data for crdb_internal.regions... writing output: debug/crdb_internal.regions.txt... done
 [cluster] retrieving SQL data for crdb_internal.schema_changes... writing output: debug/crdb_internal.schema_changes.txt... done
 [cluster] retrieving SQL data for crdb_internal.partitions... writing output: debug/crdb_internal.partitions.txt... done
 [cluster] retrieving SQL data for crdb_internal.zones... writing output: debug/crdb_internal.zones.txt... done

pkg/cli/testdata/zip/partial1_excluded

@@ -24,6 +24,7 @@ debug zip /dev/null --concurrency=1 --exclude-nodes=2 --cpu-profile-duration=0
 [cluster] retrieving SQL data for crdb_internal.kv_node_liveness... writing output: debug/crdb_internal.kv_node_liveness.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_node_status... writing output: debug/crdb_internal.kv_node_status.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_store_status... writing output: debug/crdb_internal.kv_store_status.txt... done
+[cluster] retrieving SQL data for crdb_internal.regions... writing output: debug/crdb_internal.regions.txt... done
 [cluster] retrieving SQL data for crdb_internal.schema_changes... writing output: debug/crdb_internal.schema_changes.txt... done
 [cluster] retrieving SQL data for crdb_internal.partitions... writing output: debug/crdb_internal.partitions.txt... done
 [cluster] retrieving SQL data for crdb_internal.zones... writing output: debug/crdb_internal.zones.txt... done

pkg/cli/testdata/zip/partial2

@@ -24,6 +24,7 @@ debug zip --concurrency=1 --cpu-profile-duration=0 /dev/null
 [cluster] retrieving SQL data for crdb_internal.kv_node_liveness... writing output: debug/crdb_internal.kv_node_liveness.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_node_status... writing output: debug/crdb_internal.kv_node_status.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_store_status... writing output: debug/crdb_internal.kv_store_status.txt... done
+[cluster] retrieving SQL data for crdb_internal.regions... writing output: debug/crdb_internal.regions.txt... done
 [cluster] retrieving SQL data for crdb_internal.schema_changes... writing output: debug/crdb_internal.schema_changes.txt... done
 [cluster] retrieving SQL data for crdb_internal.partitions... writing output: debug/crdb_internal.partitions.txt... done
 [cluster] retrieving SQL data for crdb_internal.zones... writing output: debug/crdb_internal.zones.txt... done

pkg/cli/testdata/zip/testzip

@@ -24,6 +24,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [cluster] retrieving SQL data for crdb_internal.kv_node_liveness... writing output: debug/crdb_internal.kv_node_liveness.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_node_status... writing output: debug/crdb_internal.kv_node_status.txt... done
 [cluster] retrieving SQL data for crdb_internal.kv_store_status... writing output: debug/crdb_internal.kv_store_status.txt... done
+[cluster] retrieving SQL data for crdb_internal.regions... writing output: debug/crdb_internal.regions.txt... done
 [cluster] retrieving SQL data for crdb_internal.schema_changes... writing output: debug/crdb_internal.schema_changes.txt... done
 [cluster] retrieving SQL data for crdb_internal.partitions... writing output: debug/crdb_internal.partitions.txt... done
 [cluster] retrieving SQL data for crdb_internal.zones... writing output: debug/crdb_internal.zones.txt... done

pkg/cli/testdata/zip/testzip_concurrent

@@ -264,6 +264,9 @@ zip
 [cluster] retrieving SQL data for crdb_internal.partitions...
 [cluster] retrieving SQL data for crdb_internal.partitions: done
 [cluster] retrieving SQL data for crdb_internal.partitions: writing output: debug/crdb_internal.partitions.txt...
+[cluster] retrieving SQL data for crdb_internal.regions...
+[cluster] retrieving SQL data for crdb_internal.regions: done
+[cluster] retrieving SQL data for crdb_internal.regions: writing output: debug/crdb_internal.regions.txt...
 [cluster] retrieving SQL data for crdb_internal.schema_changes...
 [cluster] retrieving SQL data for crdb_internal.schema_changes: done
 [cluster] retrieving SQL data for crdb_internal.schema_changes: writing output: debug/crdb_internal.schema_changes.txt...

pkg/cli/zip_cluster_wide.go

@@ -90,6 +90,7 @@ var debugZipTablesPerCluster = []string{
 	"crdb_internal.kv_node_status",
 	"crdb_internal.kv_store_status",
 
+	"crdb_internal.regions",
 	"crdb_internal.schema_changes",
 	"crdb_internal.partitions",
 	"crdb_internal.zones",

pkg/kv/kvclient/kvtenant/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//pkg/roachpb",
         "//pkg/rpc",
         "//pkg/rpc/nodedialer",
+        "//pkg/server/serverpb",
         "//pkg/util/log",
         "//pkg/util/retry",
         "@com_github_cockroachdb_errors//:errors",

pkg/kv/kvclient/kvtenant/connector.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/errors"
@@ -58,6 +59,11 @@ type Connector interface {
 	// obviates the need for SQL-only tenant processes to join the cluster-wide
 	// gossip network.
 	config.SystemConfigProvider
+
+	// Connector is capable of knowing every region in the cluster.
+	// This is necessary for region validation for zone configurations and
+	// multi-region primitives.
+	serverpb.RegionsServer
 }
 
 // ConnectorConfig encompasses the configuration required to create a Connector.

pkg/rpc/auth_tenant.go

@@ -55,6 +55,9 @@ func (a tenantAuthorizer) authorize(
 	case "/cockroach.rpc.Heartbeat/Ping":
 		return nil // no authorization
 
+	case "/cockroach.server.serverpb.Status/Regions":
+		return nil // no authorization
+
 	default:
 		return authErrorf("unknown method %q", fullMethod)
 	}

pkg/server/server.go

@@ -737,6 +737,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 		protectedtsProvider:      protectedtsProvider,
 		rangeFeedFactory:         rangeFeedFactory,
 		sqlStatusServer:          sStatus,
+		regionsServer:            sStatus,
 	})
 	if err != nil {
 		return nil, err

pkg/server/server_sql.go

@@ -249,6 +249,9 @@ type sqlServerArgs struct {
 
 	// Used to watch settings and descriptor changes.
 	rangeFeedFactory *rangefeed.Factory
+
+	// Used to query valid regions on the server.
+	regionsServer serverpb.RegionsServer
 }
 
 func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
@@ -536,6 +539,7 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 		DistSQLSrv:              distSQLServer,
 		NodesStatusServer:       cfg.nodesStatusServer,
 		SQLStatusServer:         cfg.sqlStatusServer,
+		RegionsServer:           cfg.regionsServer,
 		SessionRegistry:         cfg.sessionRegistry,
 		ContentionRegistry:      cfg.contentionRegistry,
 		SQLLivenessReader:       cfg.sqlLivenessProvider,

pkg/server/serverpb/status.go

@@ -55,6 +55,13 @@ type NodesStatusServer interface {
 	Nodes(context.Context, *NodesRequest) (*NodesResponse, error)
 }
 
+// RegionsServer is the subset of the serverpb.StatusInterface that is used
+// by the SQL system to query for available regions.
+// It is available for tenants.
+type RegionsServer interface {
+	Regions(context.Context, *RegionsRequest) (*RegionsResponse, error)
+}
+
 // OptionalNodesStatusServer returns the wrapped NodesStatusServer, if it is
 // available. If it is not, an error referring to the optionally supplied issues
 // is returned.