core: check version compatibility on RPC handshake

The RPC heartbeat now checks for version compatibility between the two
nodes. This prevents a node from establishing a gRPC connection if its
binary version is lower than the cluster version on the other end.

Fixes #18058

Release note (bug fix): Prevent RPC connections between nodes with
incompatible versions.

Author: solongordon

pkg/acceptance/cluster/dockercluster.go

@@ -47,6 +47,7 @@ import (
 	roachClient "github.com/cockroachdb/cockroach/pkg/internal/client"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/security"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/log/logflags"
@@ -811,7 +812,7 @@ func (l *DockerCluster) NewClient(ctx context.Context, i int) (*roachClient.DB,
 	rpcContext := rpc.NewContext(log.AmbientContext{Tracer: tracing.NewTracer()}, &base.Config{
 		User:        security.NodeUser,
 		SSLCertsDir: l.CertsDir,
-	}, clock, l.stopper)
+	}, clock, l.stopper, &cluster.MakeTestingClusterSettings().Version)
 	conn, err := rpcContext.GRPCDial(l.Nodes[i].Addr(ctx, DefaultTCP).String()).Connect(ctx)
 	if err != nil {
 		return nil, err

pkg/acceptance/localcluster/cluster.go

@@ -46,6 +46,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/security"
 	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
@@ -269,7 +270,7 @@ func (c *Cluster) makeNode(ctx context.Context, nodeIdx int, cfg NodeConfig) (*N
 		Insecure: true,
 	}
 	rpcCtx := rpc.NewContext(log.AmbientContext{Tracer: tracing.NewTracer()}, baseCtx,
-		hlc.NewClock(hlc.UnixNano, 0), c.stopper)
+		hlc.NewClock(hlc.UnixNano, 0), c.stopper, &cluster.MakeTestingClusterSettings().Version)
 
 	n := &Node{
 		Cfg:    cfg,
@@ -514,7 +515,7 @@ func (n *Node) Client() *client.DB {
 		return existingClient
 	}
 
-	conn, err := n.rpcCtx.GRPCDial(n.RPCAddr()).Connect(context.Background())
+	conn, err := n.rpcCtx.GRPCDialRaw(n.RPCAddr())
 	if err != nil {
 		log.Fatalf(context.Background(), "failed to initialize KV client: %s", err)
 	}
@@ -531,7 +532,7 @@ func (n *Node) StatusClient() serverpb.StatusClient {
 		return existingClient
 	}
 
-	conn, err := n.rpcCtx.GRPCDial(n.RPCAddr()).Connect(context.Background())
+	conn, err := n.rpcCtx.GRPCDialRaw(n.RPCAddr())
 	if err != nil {
 		log.Fatalf(context.Background(), "failed to initialize status client: %s", err)
 	}

pkg/acceptance/util_farmer.go

@@ -31,6 +31,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/security"
+	cl "github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
@@ -153,7 +154,7 @@ func MakeFarmer(t testing.TB, prefix string, stopper *stop.Stopper) *terrafarm.F
 		// Set a bogus address, to be used by the clock skew checks as the ID of
 		// this "node". We can't leave it blank.
 		Addr: "acceptance test client",
-	}, clientClock, stopper)
+	}, clientClock, stopper, &cl.MakeTestingClusterSettings().Version)
 	rpcContext.HeartbeatCB = func() {
 		if err := rpcContext.RemoteClocks.VerifyClockOffset(context.Background()); err != nil {
 			t.Fatal(err)

pkg/cli/start.go

@@ -938,6 +938,7 @@ func getClientGRPCConn(ctx context.Context) (*grpc.ClientConn, *hlc.Clock, func(
 		serverCfg.Config,
 		clock,
 		stopper,
+		&serverCfg.Settings.Version,
 	)
 	addr, err := addrWithDefaultHost(serverCfg.AdvertiseAddr)
 	if err != nil {

pkg/gossip/client_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/gossip/resolver"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
@@ -53,6 +54,7 @@ func newInsecureRPCContext(stopper *stop.Stopper) *rpc.Context {
 		&base.Config{Insecure: true},
 		hlc.NewClock(hlc.UnixNano, time.Nanosecond),
 		stopper,
+		&cluster.MakeTestingClusterSettings().Version,
 	)
 }
 

pkg/gossip/simulation/network.go

@@ -28,6 +28,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/gossip/resolver"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/encoding"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
@@ -77,6 +78,7 @@ func NewNetwork(stopper *stop.Stopper, nodeCount int, createResolvers bool) *Net
 		&base.Config{Insecure: true},
 		hlc.NewClock(hlc.UnixNano, time.Nanosecond),
 		n.Stopper,
+		&cluster.MakeTestingClusterSettings().Version,
 	)
 	var err error
 	n.tlsConfig, err = n.rpcContext.GetServerTLSConfig()

pkg/internal/client/client_test.go

@@ -126,7 +126,9 @@ func createTestClientForUser(
 	}
 	testutils.FillCerts(cfg)
 
-	rpcContext := rpc.NewContext(log.AmbientContext{Tracer: s.ClusterSettings().Tracer}, cfg, s.Clock(), s.Stopper())
+	rpcContext := rpc.NewContext(
+		log.AmbientContext{Tracer: s.ClusterSettings().Tracer}, cfg, s.Clock(), s.Stopper(),
+		&s.ClusterSettings().Version)
 	conn, err := rpcContext.GRPCDial(s.ServingAddr()).Connect(context.Background())
 	if err != nil {
 		t.Fatal(err)

pkg/kv/db_test.go

@@ -57,9 +57,10 @@ func createTestClientForUser(
 	ctx.User = user
 	testutils.FillCerts(&ctx)
 
-	conn, err := rpc.NewContext(
+	rpcContext := rpc.NewContext(
 		log.AmbientContext{Tracer: s.ClusterSettings().Tracer}, &ctx, s.Clock(),
-		s.Stopper()).GRPCDial(s.ServingAddr()).Connect(context.Background())
+		s.Stopper(), &s.ClusterSettings().Version)
+	conn, err := rpcContext.GRPCDial(s.ServingAddr()).Connect(context.Background())
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/kv/dist_sender_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/util"
@@ -686,6 +687,7 @@ func makeGossip(t *testing.T, stopper *stop.Stopper) (*gossip.Gossip, *hlc.Clock
 		&base.Config{Insecure: true},
 		clock,
 		stopper,
+		&cluster.MakeTestingClusterSettings().Version,
 	)
 	server := rpc.NewServer(rpcContext)
 

pkg/kv/send_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/rpc"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
@@ -59,6 +60,7 @@ func TestSendToOneClient(t *testing.T) {
 		testutils.NewNodeTestBaseContext(),
 		hlc.NewClock(hlc.UnixNano, time.Nanosecond),
 		stopper,
+		&cluster.MakeTestingClusterSettings().Version,
 	)
 	s := rpc.NewServer(rpcContext)
 	roachpb.RegisterInternalServer(s, Node(0))
@@ -127,6 +129,7 @@ func TestComplexScenarios(t *testing.T) {
 		testutils.NewNodeTestBaseContext(),
 		hlc.NewClock(hlc.UnixNano, time.Nanosecond),
 		stopper,
+		&cluster.MakeTestingClusterSettings().Version,
 	)
 
 	// TODO(bdarnell): the retryable flag is no longer used for RPC errors.