Merge #50116 #50293

50116: kv: decompose roles of Gossip dependency in DistSender r=nvanbenschoten a=nvanbenschoten

This change decomposes the roles of Gossip as a dependency in DistSender into that of a NodeDescStore, a source of node descriptors, and that of a FirstRangeProvider, a provider of information on the first range in a cluster.

This decomposition will be used to address #47909 by:
1. replacing Gossip with a TenantService as a NodeDescStore
2. providing a custom RangeDescriptorDB (also backed by a TenantService) instead of a FirstRangeProvider.

Together, these changes will allow us to remove DistSender's dependency on Gossip for SQL-only tenant processes.

The next step after this will be to introduce a TenantService that can satisfy these two dependencies (NodeDescStore and RangeDescriptorDB) and also use the new NodeDescStore-to-AddressResolver binding to replace the use of Gossip with the TenantService in nodedialer instances.

50293: server: Wrap ExternalStorage factory methods in externalStorageBuilder struct r=adityamaru a=adityamaru

Previously, we initialized the ExternalStorage factory methods on
creation of a NewServer() as all the required config params were
ready-to-use.

With future work related to user scoped storage requiring access to the
underlying storage.Engine, this change introduces a wrapper around these
factory methods. Using a builder struct allows us to split the
"creation" and "initialization" of the builder between the NewServer()
and Start() methods respectively. This allows for params which are only
initialized on server.Start() to be propogated to the builder for future
use.

This is part of a gradual refactor of the ExternalStorage factory
interface and is primarily to unblock development of #47211.

Release note: None

Co-authored-by: Nathan VanBenschoten <nvanbenschoten@gmail.com>
Co-authored-by: Aditya Maru <adityamaru@gmail.com>

pkg/gossip/gossip.go

@@ -196,7 +196,7 @@ func NewKeyNotPresentError(key string) error {
 }
 
 // AddressResolver is a thin wrapper around gossip's GetNodeIDAddress
-// that allows it to be used as a nodedialer.AddressResolver
+// that allows it to be used as a nodedialer.AddressResolver.
 func AddressResolver(gossip *Gossip) nodedialer.AddressResolver {
  return func(nodeID roachpb.NodeID) (net.Addr, error) {
  return gossip.GetNodeIDAddress(nodeID)
@@ -274,7 +274,7 @@ type Gossip struct {
  resolverAddrs map[util.UnresolvedAddr]resolver.Resolver
  bootstrapAddrs map[util.UnresolvedAddr]roachpb.NodeID
 
- localityTierMap map[string]struct{}
+ locality roachpb.Locality
 
  lastConnectivity redact.RedactableString
 
@@ -320,13 +320,10 @@ func New(
  storeMap: make(map[roachpb.StoreID]roachpb.NodeID),
  resolverAddrs: map[util.UnresolvedAddr]resolver.Resolver{},
  bootstrapAddrs: map[util.UnresolvedAddr]roachpb.NodeID{},
- localityTierMap: map[string]struct{}{},
+ locality:  locality,
  defaultZoneConfig: defaultZoneConfig,
  }
 
- for _, loc := range locality.Tiers {
- g.localityTierMap[loc.String()] = struct{}{}
- }
  stopper.AddCloser(stop.CloserFn(g.server.AmbientContext.FinishEventLog))
 
  registry.AddMetric(g.outgoing.gauge)
@@ -980,19 +977,12 @@ func (g *Gossip) getNodeIDAddressLocked(nodeID roachpb.NodeID) (*util.Unresolved
  if err != nil {
  return nil, err
  }
- for i := range nd.LocalityAddress {
- locality := &nd.LocalityAddress[i]
- if _, ok := g.localityTierMap[locality.LocalityTier.String()]; ok {
- return &locality.Address, nil
- }
- }
- return &nd.Address, nil
+ return nd.AddressForLocality(g.locality), nil
 }
 
-// getNodeIDAddressLocked looks up the SQL address of the node by ID. The mutex is
-// assumed held by the caller. This method is called externally via
-// GetNodeIDSQLAddress or internally when looking up a "distant" node address to
-// connect directly to.
+// getNodeIDAddressLocked looks up the SQL address of the node by ID. The mutex
+// is assumed held by the caller. This method is called externally via
+// GetNodeIDSQLAddress.
 func (g *Gossip) getNodeIDSQLAddressLocked(nodeID roachpb.NodeID) (*util.UnresolvedAddr, error) {
  nd, err := g.getNodeDescriptorLocked(nodeID)
  if err != nil {
@@ -1619,6 +1609,38 @@ func (g *Gossip) findClient(match func(*client) bool) *client {
  return nil
 }
 
+// A firstRangeMissingError indicates that the first range has not yet
+// been gossiped. This will be the case for a node which hasn't yet
+// joined the gossip network.
+type firstRangeMissingError struct{}
+
+// Error is part of the error interface.
+func (f firstRangeMissingError) Error() string {
+ return "the descriptor for the first range is not available via gossip"
+}
+
+// GetFirstRangeDescriptor implements kvcoord.FirstRangeProvider.
+func (g *Gossip) GetFirstRangeDescriptor() (*roachpb.RangeDescriptor, error) {
+ desc := &roachpb.RangeDescriptor{}
+ if err := g.GetInfoProto(KeyFirstRangeDescriptor, desc); err != nil {
+ return nil, firstRangeMissingError{}
+ }
+ return desc, nil
+}
+
+// OnFirstRangeChanged implements kvcoord.FirstRangeProvider.
+func (g *Gossip) OnFirstRangeChanged(cb func(*roachpb.RangeDescriptor)) {
+ g.RegisterCallback(KeyFirstRangeDescriptor, func(_ string, value roachpb.Value) {
+ ctx := context.Background()
+ desc := &roachpb.RangeDescriptor{}
+ if err := value.GetProto(desc); err != nil {
+ log.Errorf(ctx, "unable to parse gossiped first range descriptor: %s", err)
+ } else {
+ cb(desc)
+ }
+ })
+}
+
 // MakeExposedGossip initializes a DeprecatedGossip instance which exposes a
 // wrapped Gossip instance via Optional(). This is used on SQL servers running
 // inside of a KV server (i.e. single-tenant deployments).

pkg/gossip/keys.go

@@ -28,27 +28,27 @@ const separator = ":"
 // Constants for gossip keys.
 const (
  // KeyClusterID is the unique UUID for this Cockroach cluster.
- // The value is a string UUID for the cluster.  The cluster ID is
+ // The value is a string UUID for the cluster. The cluster ID is
  // gossiped by all nodes that contain a replica of the first range,
  // and it serves as a check for basic gossip connectivity. The
  // Gossip.Connected channel is closed when we see this key.
  KeyClusterID = "cluster-id"
 
  // KeyStorePrefix is the key prefix for gossiping stores in the network.
- // The suffix is a store ID and the value is roachpb.StoreDescriptor.
+ // The suffix is a store ID and the value is a roachpb.StoreDescriptor.
  KeyStorePrefix = "store"
 
- // KeyNodeIDPrefix is the key prefix for gossiping node id
- // addresses. The actual key is suffixed with the decimal
- // representation of the node id and the value is the host:port
- // string address of the node. E.g. node:1 => 127.0.0.1:24001
+ // KeyNodeIDPrefix is the key prefix for gossiping node id addresses.
+ // The actual key is suffixed with the decimal representation of the
+ // node id (e.g. 'node:1') and the value is a roachpb.NodeDescriptor.
  KeyNodeIDPrefix = "node"
 
- // KeyHealthAlertPrefix is the key prefix for gossiping health alerts. The
- // value is a proto of type HealthCheckResult.
+ // KeyHealthAlertPrefix is the key prefix for gossiping health alerts.
+ // The value is a proto of type HealthCheckResult.
  KeyNodeHealthAlertPrefix = "health-alert"
 
- // KeyNodeLivenessPrefix is the key prefix for gossiping node liveness info.
+ // KeyNodeLivenessPrefix is the key prefix for gossiping node liveness
+ // info.
  KeyNodeLivenessPrefix = "liveness"
 
  // KeySentinel is a key for gossip which must not expire or
@@ -57,10 +57,9 @@ const (
  // the range lease for the first range.
  KeySentinel = "sentinel"
 
- // KeyFirstRangeDescriptor is the descriptor for the "first"
- // range. The "first" range contains the meta1 key range, the first
- // level of the bi-level key addressing scheme. The value is a slice
- // of storage.Replica structs.
+ // KeyFirstRangeDescriptor is the descriptor for the "first" range. The
+ // "first" range contains the meta1 key range, the first level of the
+ // bi-level key addressing scheme. The value is a roachpb.RangeDescriptor.
  KeyFirstRangeDescriptor = "first-range"
 
  // KeySystemConfig is the gossip key for the system DB span.