Fix multi-cluster raft tests that use TCP networking (#24894)

helper/testhelpers/teststorage/teststorage.go

@@ -236,12 +236,19 @@ func FileBackendSetup(conf *vault.CoreConfig, opts *vault.TestClusterOptions) {
 func RaftBackendSetup(conf *vault.CoreConfig, opts *vault.TestClusterOptions) {
 	opts.KeepStandbysSealed = true
 	var bridge *raft.ClusterAddrBridge
-	if !opts.InmemClusterLayers && opts.ClusterLayers == nil {
-		bridge = raft.NewClusterAddrBridge()
-	}
-	conf.ClusterAddrBridge = bridge
 	opts.PhysicalFactory = func(t testing.T, coreIdx int, logger hclog.Logger, conf map[string]interface{}) *vault.PhysicalBackendBundle {
-		return MakeRaftBackend(t, coreIdx, logger, conf, bridge)
+		// The same PhysicalFactory can be shared across multiple clusters.
+		// The coreIdx == 0 check ensures that each time a new cluster is setup,
+		// when setting up its first node we create a new ClusterAddrBridge.
+		if !opts.InmemClusterLayers && opts.ClusterLayers == nil && coreIdx == 0 {
+			bridge = raft.NewClusterAddrBridge()
+		}
+		bundle := MakeRaftBackend(t, coreIdx, logger, conf, bridge)
+		bundle.MutateCoreConfig = func(conf *vault.CoreConfig) {
+			logger.Trace("setting bridge", "idx", coreIdx, "bridge", fmt.Sprintf("%p", bridge))
+			conf.ClusterAddrBridge = bridge
+		}
+		return bundle
 	}
 	opts.SetupFunc = func(t testing.T, c *vault.TestCluster) {
 		if opts.NumCores != 1 {

vault/testing.go

@@ -33,7 +33,6 @@ import (
 	"github.com/hashicorp/go-cleanhttp"
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/reloadutil"
-	raftlib "github.com/hashicorp/raft"
 	kv "github.com/hashicorp/vault-plugin-secrets-kv"
 	"github.com/mitchellh/copystructure"
 	"github.com/mitchellh/go-testing-interface"
@@ -1041,6 +1040,12 @@ type PhysicalBackendBundle struct {
 	Backend   physical.Backend
 	HABackend physical.HABackend
 	Cleanup   func()
+	// MutateCoreConfig is invoked when applying the bundle to core config
+	// during core creation.  We do it this way rather than providing the input
+	// directly as part of the CoreConfig passed in to NewTestCluster so that
+	// the same input conf and opts can be used for multiple clusters, but at
+	// the same time can have per-cluster configuration specified by the backend.
+	MutateCoreConfig func(conf *CoreConfig)
 }
 
 type HandlerHandler interface {
@@ -1104,14 +1109,6 @@ type TestClusterOptions struct {
 	// built using the inmem implementation.
 	InmemClusterLayers bool
 
-	// RaftAddressProvider is used to set the raft ServerAddressProvider on
-	// each core.
-	//
-	// If SkipInit is true, then RaftAddressProvider has no effect.
-	// RaftAddressProvider should only be specified if the underlying physical
-	// storage is Raft.
-	RaftAddressProvider raftlib.ServerAddressProvider
-
 	CoreMetricSinkProvider func(clusterName string) (*metricsutil.ClusterMetricSink, *metricsutil.MetricsHelper)
 
 	PhysicalFactoryConfig        map[string]interface{}
@@ -1863,6 +1860,10 @@ func (testCluster *TestCluster) newCore(t testing.T, idx int, coreConfig *CoreCo
 			if physBundle.Cleanup != nil {
 				cleanupFunc = physBundle.Cleanup
 			}
+
+			if physBundle.MutateCoreConfig != nil {
+				physBundle.MutateCoreConfig(&localConfig)
+			}
 		}
 	}