roachtest: allow tests to specify a cockroach binary to use

pkg/cmd/roachtest/cluster.go

@@ -40,6 +40,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/tests"
 	"github.com/cockroachdb/cockroach/pkg/roachprod"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/config"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
@@ -1749,16 +1750,25 @@ func (c *clusterImpl) PutE(
 	return errors.Wrap(roachprod.Put(ctx, l, c.MakeNodes(nodes...), src, dest, true /* useTreeDist */), "cluster.PutE")
 }
 
-// PutDefaultCockroach uploads the cockroach binary passed in the
-// command line to `test.DefaultCockroachPath` in every node in the
-// cluster. This binary is used by the test runner to collect failure
-// artifacts since tests are free to upload the cockroach binary they
-// use to any location they desire.
-func (c *clusterImpl) PutDefaultCockroach(
-	ctx context.Context, l *logger.Logger, cockroachPath string,
-) error {
-	c.status("uploading default cockroach binary to nodes")
-	return c.PutE(ctx, l, cockroachPath, test.DefaultCockroachPath, c.All())
+// PutCockroach checks if a test specifies a cockroach binary to upload to all
+// nodes in the cluster. By default, we randomly upload a binary with or without
+// runtime assertions enabled. Note that we upload to all nodes even if they
+// don't use the binary, so that the test runner can always fetch logs.
+func (c *clusterImpl) PutCockroach(ctx context.Context, l *logger.Logger, t *testImpl) error {
+	switch t.spec.CockroachBinary {
+	case registry.RandomizedCockroach:
+		if tests.UsingRuntimeAssertions(t) {
+			t.l.Printf("To reproduce the same set of metamorphic constants, run this test with %s=%d", test.EnvAssertionsEnabledSeed, c.cockroachRandomSeed())
+		}
+		return c.PutE(ctx, l, t.Cockroach(), test.DefaultCockroachPath, c.All())
+	case registry.StandardCockroach:
+		return c.PutE(ctx, l, t.StandardCockroach(), test.DefaultCockroachPath, c.All())
+	case registry.RuntimeAssertionsCockroach:
+		t.l.Printf("To reproduce the same set of metamorphic constants, run this test with %s=%d", test.EnvAssertionsEnabledSeed, c.cockroachRandomSeed())
+		return c.PutE(ctx, l, t.RuntimeAssertionsCockroach(), test.DefaultCockroachPath, c.All())
+	default:
+		return errors.Errorf("Specified cockroach binary does not exist.")
+	}
 }
 
 // PutLibraries inserts the specified libraries, by name, into all nodes on the cluster

pkg/cmd/roachtest/github.go

@@ -259,7 +259,16 @@ func (g *githubIssues) MaybePost(t *testImpl, l *logger.Logger, message string)
 		return nil
 	}
 
-	postRequest, err := g.createPostRequest(t.Name(), t.start, t.end, t.spec, t.firstFailure(), message, tests.UsingRuntimeAssertions(t))
+	var metamorphicBuild bool
+	switch t.spec.CockroachBinary {
+	case registry.StandardCockroach:
+		metamorphicBuild = false
+	case registry.RuntimeAssertionsCockroach:
+		metamorphicBuild = true
+	default:
+		metamorphicBuild = tests.UsingRuntimeAssertions(t)
+	}
+	postRequest, err := g.createPostRequest(t.Name(), t.start, t.end, t.spec, t.firstFailure(), message, metamorphicBuild)
 	if err != nil {
 		return err
 	}

pkg/cmd/roachtest/registry/test_spec.go

@@ -132,6 +132,13 @@ type TestSpec struct {
 	// ExtraLabels are test-specific labels that will be added to the Github
 	// issue created when a failure occurs, in addition to default labels.
 	ExtraLabels []string
+
+	// CockroachBinary is the cockroach binary that will be uploaded
+	// to every node in the cluster at the start of the test. We upload to
+	// every node so that we can fetch logs in the case of a failure.
+	// If one is not specified, the default behavior is to upload
+	// a binary with the crdb_test flag randomly enabled or disabled.
+	CockroachBinary ClusterCockroachBinary
 }
 
 // PostValidation is a type of post-validation that runs after a test completes.
@@ -426,3 +433,13 @@ func setToString(validValues []string, m map[string]struct{}) string {
 	}
 	return strings.Join(elems, ",")
 }
+
+// ClusterCockroachBinary specifies the type of cockroach binaries that
+// can be uploaded to the cluster.
+type ClusterCockroachBinary int
+
+const (
+	RandomizedCockroach ClusterCockroachBinary = iota
+	StandardCockroach
+	RuntimeAssertionsCockroach
+)

pkg/cmd/roachtest/test/test_interface.go

@@ -18,7 +18,7 @@ import (
 // DefaultCockroachPath is the path where the binary passed to the
 // `--cockroach` flag will be made available in every node in the
 // cluster.
-const DefaultCockroachPath = "./cockroach-default"
+const DefaultCockroachPath = "./cockroach"
 
 // EnvAssertionsEnabledSeed is the name of the environment variable
 // that, when set, causes roachtest to use a binary with runtime

pkg/cmd/roachtest/test_runner.go

@@ -746,7 +746,7 @@ func (r *testRunner) runWorker(
 
 			var setupErr error
 			if c.spec.NodeCount > 0 { // skip during tests
-				setupErr = c.PutDefaultCockroach(ctx, l, t.Cockroach())
+				setupErr = c.PutCockroach(ctx, l, t)
 			}
 			if setupErr == nil {
 				setupErr = c.PutLibraries(ctx, "./lib", t.spec.NativeLibs)

pkg/cmd/roachtest/tests/activerecord.go

@@ -48,7 +48,6 @@ func registerActiveRecord(r registry.Registry) {
 		}
 		node := c.Node(1)
 		t.Status("setting up cockroach")
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 		c.Start(ctx, t.L(), option.DefaultStartOptsInMemory(), install.MakeClusterSettings(), c.All())
 
 		version, err := fetchCockroachVersion(ctx, t.L(), c, node[0])

pkg/cmd/roachtest/tests/admission_control_database_drop.go

@@ -197,7 +197,6 @@ func registerDatabaseDrop(r registry.Registry) {
 			// test and use disk snapshots?
 			runTPCE(ctx, t, c, tpceOptions{
 				start: func(ctx context.Context, t test.Test, c cluster.Cluster) {
-					c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 					startOpts := option.DefaultStartOptsNoBackups()
 					settings := install.MakeClusterSettings(install.NumRacksOption(crdbNodes))
 					if err := c.StartE(ctx, t.L(), startOpts, settings, c.Range(1, crdbNodes)); err != nil {

pkg/cmd/roachtest/tests/admission_control_elastic_io.go

@@ -68,7 +68,6 @@ func registerElasticIO(r registry.Registry) {
 			require.NoError(t, err)
 			statCollector := clusterstats.NewStatsCollector(ctx, promClient)
 
-			c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, crdbNodes))
 			c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(workAndPromNode))
 			startOpts := option.DefaultStartOptsNoBackups()
 			startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs,

pkg/cmd/roachtest/tests/admission_control_follower_overload.go

@@ -147,7 +147,6 @@ func runAdmissionControlFollowerOverload(
 	phaseDuration := time.Hour
 
 	nodes := c.Range(1, 3)
-	c.Put(ctx, t.Cockroach(), "cockroach")
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), nodes)
 	db := c.Conn(ctx, t.L(), 1)
 	require.NoError(t, WaitFor3XReplication(ctx, t, db))

pkg/cmd/roachtest/tests/admission_control_index_backfill.go

@@ -151,7 +151,6 @@ func registerIndexBackfill(r registry.Registry) {
 			// large index backfills while it's running.
 			runTPCE(ctx, t, c, tpceOptions{
 				start: func(ctx context.Context, t test.Test, c cluster.Cluster) {
-					c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 					startOpts := option.DefaultStartOptsNoBackups()
 					settings := install.MakeClusterSettings(install.NumRacksOption(crdbNodes))
 					if err := c.StartE(ctx, t.L(), startOpts, settings, c.Range(1, crdbNodes)); err != nil {

pkg/cmd/roachtest/tests/admission_control_index_overload.go

@@ -48,7 +48,6 @@ func registerIndexOverload(r registry.Registry) {
 			crdbNodes := c.Spec().NodeCount - 1
 			workloadNode := c.Spec().NodeCount
 
-			c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 			c.Start(ctx, t.L(), option.DefaultStartOptsNoBackups(), install.MakeClusterSettings(), c.Range(1, crdbNodes))
 
 			{

pkg/cmd/roachtest/tests/admission_control_intent_resolution.go

@@ -67,7 +67,6 @@ func registerIntentResolutionOverload(r registry.Registry) {
 			require.NoError(t, err)
 			statCollector := clusterstats.NewStatsCollector(ctx, promClient)
 
-			c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, crdbNodes))
 			startOpts := option.DefaultStartOptsNoBackups()
 			startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs,
 				"--vmodule=io_load_listener=2")

pkg/cmd/roachtest/tests/admission_control_multi_store_overload.go

@@ -26,7 +26,6 @@ import (
 func registerMultiStoreOverload(r registry.Registry) {
 	runKV := func(ctx context.Context, t test.Test, c cluster.Cluster) {
 		nodes := c.Spec().NodeCount - 1
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes))
 		c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1))
 		startOpts := option.DefaultStartOptsNoBackups()
 		startOpts.RoachprodOpts.StoreCount = 2

pkg/cmd/roachtest/tests/admission_control_multitenant_fairness.go

@@ -144,7 +144,6 @@ func runMultiTenantFairness(
 	}
 
 	t.L().Printf("starting cockroach securely (<%s)", time.Minute)
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	c.Start(ctx, t.L(),
 		option.DefaultStartOptsNoBackups(),
 		install.MakeClusterSettings(install.SecureOption(true)),

pkg/cmd/roachtest/tests/admission_control_snapshot_overload.go

@@ -52,7 +52,6 @@ func registerSnapshotOverload(r registry.Registry) {
 				t.Fatalf("expected at least 4 nodes, found %d", c.Spec().NodeCount)
 			}
 
-			c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 			crdbNodes := c.Spec().NodeCount - 1
 			workloadNode := crdbNodes + 1
 			for i := 1; i <= crdbNodes; i++ {

pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go

@@ -198,7 +198,6 @@ func registerTPCCSevereOverload(r registry.Registry) {
 			roachNodes := c.Range(1, c.Spec().NodeCount-1)
 			workloadNode := c.Spec().NodeCount
 
-			c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 			c.Start(ctx, t.L(), option.DefaultStartOptsNoBackups(), install.MakeClusterSettings(), roachNodes)
 
 			t.Status("initializing (~1h)")

pkg/cmd/roachtest/tests/allocation_bench.go

@@ -274,7 +274,6 @@ func setupAllocationBench(
 	ctx context.Context, t test.Test, c cluster.Cluster, spec allocationBenchSpec,
 ) (clusterstats.StatCollector, func(context.Context)) {
 	workloadNode := c.Spec().NodeCount
-	c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 	c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(workloadNode))
 	t.Status("starting cluster")
 	for i := 1; i <= spec.nodes; i++ {

pkg/cmd/roachtest/tests/allocator.go

@@ -37,8 +37,6 @@ const allocatorStableSeconds = 120
 
 func registerAllocator(r registry.Registry) {
 	runAllocator := func(ctx context.Context, t test.Test, c cluster.Cluster, start int, maxStdDev float64) {
-		c.Put(ctx, t.Cockroach(), "./cockroach")
-
 		// Put away one node to be the stats collector.
 		nodes := c.Spec().NodeCount - 1
 
@@ -346,7 +344,6 @@ func runWideReplication(ctx context.Context, t test.Test, c cluster.Cluster) {
 		t.Fatalf("9-node cluster required")
 	}
 
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	startOpts := option.DefaultStartOpts()
 	startOpts.RoachprodOpts.ExtraArgs = []string{"--vmodule=replicate_queue=6"}
 	settings := install.MakeClusterSettings()

pkg/cmd/roachtest/tests/alterpk.go

@@ -30,7 +30,6 @@ func registerAlterPK(r registry.Registry) {
 		roachNodes := c.Range(1, c.Spec().NodeCount-1)
 		loadNode := c.Node(c.Spec().NodeCount)
 		t.Status("copying binaries")
-		c.Put(ctx, t.Cockroach(), "./cockroach", roachNodes)
 		c.Put(ctx, t.DeprecatedWorkload(), "./workload", loadNode)
 
 		t.Status("starting cockroach nodes")

pkg/cmd/roachtest/tests/asyncpg.go

@@ -46,7 +46,6 @@ func registerAsyncpg(r registry.Registry) {
 		}
 		node := c.Node(1)
 		t.Status("setting up cockroach")
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 
 		// This test assumes that multiple_active_portals_enabled is false, but through
 		// metamorphic constants, it is possible for them to be enabled. We disable

pkg/cmd/roachtest/tests/awsdms.go

@@ -563,7 +563,6 @@ func setupAWSDMS(
 func setupCockroachDBCluster(ctx context.Context, t test.Test, c cluster.Cluster) func() error {
 	return func() error {
 		t.L().Printf("setting up cockroach")
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 		c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All())
 
 		db := c.Conn(ctx, t.L(), 1)

pkg/cmd/roachtest/tests/backup.go

@@ -96,8 +96,6 @@ func importBankDataSplit(
 ) string {
 	dest := destinationName(c)
 
-	cockroach := t.Cockroach()
-	c.Put(ctx, cockroach, "./cockroach")
 	c.Put(ctx, t.DeprecatedWorkload(), "./workload")
 
 	// NB: starting the cluster creates the logs dir as a side effect,
@@ -745,7 +743,6 @@ func runBackupMVCCRangeTombstones(
 	ctx context.Context, t test.Test, c cluster.Cluster, config mvccRangeTombstoneConfig,
 ) {
 	if !config.skipClusterSetup {
-		c.Put(ctx, t.Cockroach(), "./cockroach")
 		c.Put(ctx, t.DeprecatedWorkload(), "./workload") // required for tpch
 		c.Start(ctx, t.L(), maybeUseMemoryBudget(t, 50), install.MakeClusterSettings())
 	}

pkg/cmd/roachtest/tests/build_info.go

@@ -25,7 +25,6 @@ import (
 
 // RunBuildInfo is a test that sanity checks the build info.
 func RunBuildInfo(ctx context.Context, t test.Test, c cluster.Cluster) {
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings())
 
 	var details serverpb.DetailsResponse
@@ -63,8 +62,6 @@ func RunBuildAnalyze(ctx context.Context, t test.Test, c cluster.Cluster) {
 		t.Skip("local execution not supported")
 	}
 
-	c.Put(ctx, t.Cockroach(), "./cockroach")
-
 	// 1. Check for executable stack.
 	//
 	// Executable stack memory is a security risk (not a vulnerability

pkg/cmd/roachtest/tests/cancel.go

@@ -44,7 +44,6 @@ import (
 // replaced with unit tests.
 func registerCancel(r registry.Registry) {
 	runCancel := func(ctx context.Context, t test.Test, c cluster.Cluster, tpchQueriesToRun []int, useDistsql bool) {
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
 		c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All())
 
 		m := c.NewMonitor(ctx, c.All())

pkg/cmd/roachtest/tests/cdc.go

@@ -619,10 +619,6 @@ func newCDCTester(ctx context.Context, t test.Test, c cluster.Cluster) cdcTester
 
 	settings.Env = append(settings.Env, envVars...)
 
-	// Allow cockroach with runtime assertions enabled unless this is a
-	// performance test.
-	cockroach := t.Cockroach()
-	c.Put(ctx, cockroach, "./cockroach")
 	c.Start(ctx, t.L(), startOpts, settings, tester.crdbNodes)
 	c.Put(ctx, t.DeprecatedWorkload(), "./workload", tester.workloadNode)
 	tester.startGrafana()
@@ -666,7 +662,6 @@ func runCDCBank(ctx context.Context, t test.Test, c cluster.Cluster) {
 	c.Run(ctx, c.All(), `mkdir -p logs`)
 
 	crdbNodes, workloadNode, kafkaNode := c.Range(1, c.Spec().NodeCount-1), c.Node(c.Spec().NodeCount), c.Node(c.Spec().NodeCount)
-	c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes)
 	c.Put(ctx, t.DeprecatedWorkload(), "./workload", workloadNode)
 	startOpts := option.DefaultStartOpts()
 	startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs,
@@ -835,7 +830,6 @@ func runCDCBank(ctx context.Context, t test.Test, c cluster.Cluster) {
 // compatibility within a topic).
 func runCDCSchemaRegistry(ctx context.Context, t test.Test, c cluster.Cluster) {
 	crdbNodes, kafkaNode := c.Node(1), c.Node(1)
-	c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes)
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes)
 	kafka := kafkaManager{
 		t:     t,
@@ -951,7 +945,6 @@ func runCDCKafkaAuth(ctx context.Context, t test.Test, c cluster.Cluster) {
 	}
 
 	crdbNodes, kafkaNode := c.Range(1, lastCrdbNode), c.Node(c.Spec().NodeCount)
-	c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes)
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes)
 
 	kafka := kafkaManager{

pkg/cmd/roachtest/tests/cdc_bench.go

@@ -260,7 +260,6 @@ func runCDCBenchScan(
 		t.Fatalf("unknown protocol %q", protocol)
 	}
 
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	c.Start(ctx, t.L(), opts, settings, nData)
 	m := c.NewMonitor(ctx, nData.Merge(nCoord))
 
@@ -432,7 +431,6 @@ func runCDCBenchWorkload(
 		t.Fatalf("unknown server type %q", server)
 	}
 
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	c.Start(ctx, t.L(), opts, settings, nData)
 	m := c.NewMonitor(ctx, nData.Merge(nCoord))
 

pkg/cmd/roachtest/tests/clearrange.go

@@ -63,8 +63,6 @@ func registerClearRange(r registry.Registry) {
 }
 
 func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressiveChecks bool) {
-	c.Put(ctx, t.Cockroach(), "./cockroach")
-
 	t.Status("restoring fixture")
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings())
 	m := c.NewMonitor(ctx)

pkg/cmd/roachtest/tests/cli.go

@@ -25,7 +25,6 @@ import (
 )
 
 func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) {
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 3))
 
 	db := c.Conn(ctx, t.L(), 1)

pkg/cmd/roachtest/tests/clock_jump_crash.go

@@ -36,10 +36,6 @@ func runClockJump(ctx context.Context, t test.Test, c cluster.Cluster, tc clockJ
 		t.Fatal(err)
 	}
 
-	if err := c.RunE(ctx, c.Node(1), "test -x ./cockroach"); err != nil {
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
-	}
-	c.Wipe(ctx, false /* preserveCerts */)
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings())
 
 	db := c.Conn(ctx, t.L(), c.Spec().NodeCount)

pkg/cmd/roachtest/tests/clock_monotonic.go

@@ -38,10 +38,6 @@ func runClockMonotonicity(
 		t.Fatal(err)
 	}
 
-	if err := c.RunE(ctx, c.Node(1), "test -x ./cockroach"); err != nil {
-		c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
-	}
-	c.Wipe(ctx, false /* preserveCerts */)
 	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings())
 
 	db := c.Conn(ctx, t.L(), c.Spec().NodeCount)

pkg/cmd/roachtest/tests/cluster_init.go

@@ -31,8 +31,6 @@ import (
 )
 
 func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) {
-	c.Put(ctx, t.Cockroach(), "./cockroach")
-
 	// We start all nodes with the same join flags and then issue an "init"
 	// command to one of the nodes. We do this twice, since roachtest has some
 	// special casing for the first node in a cluster (the join flags of all nodes

pkg/cmd/roachtest/tests/cluster_to_cluster.go

@@ -493,7 +493,6 @@ func (rd *replicationDriver) setupC2C(
 		require.NotEqual(t, "", rd.rs.multiregion.workloadNodeZone)
 	}
 
-	c.Put(ctx, t.Cockroach(), "./cockroach")
 	srcCluster := c.Range(1, rd.rs.srcNodes)
 	dstCluster := c.Range(rd.rs.srcNodes+1, rd.rs.srcNodes+rd.rs.dstNodes)
 	workloadNode := c.Node(rd.rs.srcNodes + rd.rs.dstNodes + 1)