cmd: replacing release-20.1 version of roachprod, roachtest, workload…

…(ccl)

This change does the following:

- Deletes release-20.1 roachprod, roachtest, workload and workloadccl
  packages.

- Adds the above packages from master.

- Make all the above packages compatible with 20.1. I have highlighted
  some of the major changes in each package and will inline links to why
  they were changed on master. I took some decisions on what required a
  backport and what looked too invasive, but am open to suggestions
  here.

roachtest:
- Deleted roachtests with min version 20.2.0 namely:
	- backup/KMS
	- join-init/mixed
	- disk_full
	- multitenant acceptance
	- tpchvec/bench
	- tpchvec/perf_no_stats
- Backported `tpcdsvec` roachtest

workload:
- Backported `schemachange` workload
- deleted geospatial.go
- switched all `rowenc` pkg references back to `sqlbase`
- type representation in 20.2 differed from what was supported in 20.1.
  All workloads had to be switched back to using `colTypes`

workloadccl:
- More type representation related changes
- Set --deprecated-fk-indexes to be always true for tpcc workload

Release note: None

pkg/ccl/workloadccl/allccl/all.go

@@ -19,6 +19,7 @@ import (
 	_ "github.com/cockroachdb/cockroach/pkg/workload/debug"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/examples"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/indexes"
+	_ "github.com/cockroachdb/cockroach/pkg/workload/interleavebench"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/interleavedpartitioned"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/jsonload"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/kv"
@@ -28,6 +29,7 @@ import (
 	_ "github.com/cockroachdb/cockroach/pkg/workload/querylog"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/queue"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/rand"
+	_ "github.com/cockroachdb/cockroach/pkg/workload/schemachange"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/sqlsmith"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/tpcc"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/tpccchecks"

pkg/ccl/workloadccl/allccl/all_test.go

@@ -11,6 +11,7 @@ package allccl
 import (
 	"context"
 	"encoding/binary"
+	"fmt"
 	"hash"
 	"hash/fnv"
 	"math"
@@ -22,13 +23,15 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/col/coldata"
 	"github.com/cockroachdb/cockroach/pkg/col/coltypes"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/bufalloc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/workload"
 	"github.com/cockroachdb/cockroach/pkg/workload/workloadsql"
-	"github.com/pkg/errors"
+	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
 )
 
@@ -78,9 +81,10 @@ func TestAllRegisteredImportFixture(t *testing.T) {
 		}
 
 		t.Run(meta.Name, func(t *testing.T) {
-			if bigInitialData(meta) && testing.Short() {
-				t.Skipf(`%s loads a lot of data`, meta.Name)
+			if bigInitialData(meta) {
+				skip.UnderShort(t, fmt.Sprintf(`%s loads a lot of data`, meta.Name))
 			}
+			defer log.Scope(t).Close(t)
 
 			ctx := context.Background()
 			s, db, _ := serverutils.StartServer(t, base.TestServerArgs{
@@ -127,7 +131,8 @@ func TestAllRegisteredSetup(t *testing.T) {
 		case `roachmart`:
 			// TODO(dan): It'd be nice to test this with the default flags. For now,
 			// this is better than nothing.
-			if err := gen.(workload.Flagser).Flags().Parse([]string{
+			flags := gen.(workload.Flagser).Flags()
+			if err := flags.Parse([]string{
 				`--users=10`, `--orders=100`, `--partition=false`,
 			}); err != nil {
 				t.Fatal(err)
@@ -138,6 +143,7 @@ func TestAllRegisteredSetup(t *testing.T) {
 		}
 
 		t.Run(meta.Name, func(t *testing.T) {
+			defer log.Scope(t).Close(t)
 			ctx := context.Background()
 			s, db, _ := serverutils.StartServer(t, base.TestServerArgs{
 				UseDatabase: "d",
@@ -165,6 +171,7 @@ func TestAllRegisteredSetup(t *testing.T) {
 
 func TestConsistentSchema(t *testing.T) {
 	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
 	// Test that the table schemas are consistent when the workload is created
 	// multiple times with the same seed.
 
@@ -188,7 +195,7 @@ func hashTableInitialData(
 	h hash.Hash, data workload.BatchedTuples, a *bufalloc.ByteAllocator,
 ) error {
 	var scratch [8]byte
-	b := coldata.NewMemBatchWithSize(nil, 0)
+	b := coldata.NewMemBatchWithSize(nil /* types */, 0 /* size */)
 	for batchIdx := 0; batchIdx < data.NumBatches; batchIdx++ {
 		*a = (*a)[:0]
 		data.FillBatch(batchIdx, b, a)
@@ -237,9 +244,7 @@ func TestDeterministicInitialData(t *testing.T) {
 
 	// There are other tests that run initial data generation under race, so we
 	// don't get anything from running this one under race as well.
-	if util.RaceEnabled {
-		t.Skip(`uninteresting under race`)
-	}
+	skip.UnderRace(t, "uninteresting under race")
 
 	// Hardcode goldens for the fingerprint of the initial data of generators with
 	// default flags. This lets us opt in generators known to be deterministic and
@@ -263,7 +268,7 @@ func TestDeterministicInitialData(t *testing.T) {
 		`startrek`:   0xa0249fbdf612734c,
 		`tpcc`:       0xab32e4f5e899eb2f,
 		`tpch`:       0xdd952207e22aa577,
-		`ycsb`:       0x85dd34d8c07fd808,
+		`ycsb`:       0x1244ea1c29ef67f6,
 	}
 
 	var a bufalloc.ByteAllocator
@@ -277,8 +282,8 @@ func TestDeterministicInitialData(t *testing.T) {
 			continue
 		}
 		t.Run(meta.Name, func(t *testing.T) {
-			if bigInitialData(meta) && testing.Short() {
-				t.Skipf(`%s involves a lot of data`, meta.Name)
+			if bigInitialData(meta) {
+				skip.UnderShort(t, fmt.Sprintf(`%s involves a lot of data`, meta.Name))
 			}
 
 			h := fnv.New64()

pkg/ccl/workloadccl/bench_test.go

@@ -16,6 +16,7 @@ import (
 	_ "github.com/cockroachdb/cockroach/pkg/ccl"
 	"github.com/cockroachdb/cockroach/pkg/ccl/workloadccl"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/workload"
 	"github.com/cockroachdb/cockroach/pkg/workload/tpcc"
@@ -48,9 +49,7 @@ func benchmarkImportFixture(b *testing.B, gen workload.Generator) {
 }
 
 func BenchmarkImportFixture(b *testing.B) {
-	if testing.Short() {
-		b.Skip("skipping long benchmark")
-	}
+	skip.UnderShort(b, "skipping long benchmark")
 
 	b.Run(`tpcc/warehouses=1`, func(b *testing.B) {
 		benchmarkImportFixture(b, tpcc.FromWarehouses(1))

pkg/ccl/workloadccl/cliccl/fixtures.go

@@ -22,7 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/workload"
 	workloadcli "github.com/cockroachdb/cockroach/pkg/workload/cli"
 	"github.com/cockroachdb/cockroach/pkg/workload/workloadsql"
-	"github.com/pkg/errors"
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"google.golang.org/api/option"
@@ -47,6 +47,7 @@ func config() workloadccl.FixtureConfig {
 		config.BillingProject = *gcsBillingProjectOverride
 	}
 	config.CSVServerURL = *fixturesMakeImportCSVServerURL
+	config.TableStats = *fixturesMakeTableStats
 	return config
 }
 
@@ -91,6 +92,10 @@ var fixturesMakeFilesPerNode = fixturesMakeCmd.PersistentFlags().Int(
 	`files-per-node`, 1,
 	`number of file URLs to generate per node when using csv-server`)
 
+var fixturesMakeTableStats = fixturesMakeCmd.PersistentFlags().Bool(
+	`table-stats`, true,
+	`generate full table statistics for all tables`)
+
 var fixturesImportFilesPerNode = fixturesImportCmd.PersistentFlags().Int(
 	`files-per-node`, 1,
 	`number of file URLs to generate per node`)

pkg/ccl/workloadccl/fixture.go

@@ -28,7 +28,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/cockroach/pkg/workload"
-	"github.com/pkg/errors"
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/pflag"
 	"google.golang.org/api/iterator"
 )
@@ -60,6 +60,10 @@ type FixtureConfig struct {
 	// storage requests. This is required to be set if using a "requestor pays"
 	// bucket.
 	BillingProject string
+
+	// If TableStats is true, CREATE STATISTICS is called on all tables before
+	// creating the fixture.
+	TableStats bool
 }
 
 func (s FixtureConfig) objectPathToURI(folder string) string {
@@ -142,7 +146,7 @@ func GetFixture(
 
 			fixtureFolder := generatorToGCSFolder(config, gen)
 			_, err := b.Objects(ctx, &storage.Query{Prefix: fixtureFolder, Delimiter: `/`}).Next()
-			if err == iterator.Done {
+			if errors.Is(err, iterator.Done) {
 				notFound = true
 				return errors.Errorf(`fixture not found: %s`, fixtureFolder)
 			} else if err != nil {
@@ -153,7 +157,7 @@ func GetFixture(
 			for _, table := range gen.Tables() {
 				tableFolder := filepath.Join(fixtureFolder, table.Name)
 				_, err := b.Objects(ctx, &storage.Query{Prefix: tableFolder, Delimiter: `/`}).Next()
-				if err == iterator.Done {
+				if errors.Is(err, iterator.Done) {
 					return errors.Errorf(`fixture table not found: %s`, tableFolder)
 				} else if err != nil {
 					return err
@@ -279,6 +283,28 @@ func MakeFixture(
 		return Fixture{}, err
 	}
 
+	if config.TableStats {
+		// Clean up any existing statistics.
+		_, err := sqlDB.Exec("DELETE FROM system.table_statistics WHERE true")
+		if err != nil {
+			return Fixture{}, errors.Wrapf(err, "while deleting table statistics")
+		}
+		g := ctxgroup.WithContext(ctx)
+		for _, t := range gen.Tables() {
+			t := t
+			g.Go(func() error {
+				log.Infof(ctx, "Creating table stats for %s", t.Name)
+				_, err := sqlDB.Exec(fmt.Sprintf(
+					`CREATE STATISTICS pre_backup FROM "%s"."%s"`, dbName, t.Name,
+				))
+				return err
+			})
+		}
+		if err := g.Wait(); err != nil {
+			return Fixture{}, err
+		}
+	}
+
 	g := ctxgroup.WithContext(ctx)
 	for _, t := range gen.Tables() {
 		t := t
@@ -554,11 +580,12 @@ func RestoreFixture(
 		g.GoCtx(func(ctx context.Context) error {
 			start := timeutil.Now()
 			importStmt := fmt.Sprintf(`RESTORE %s.%s FROM $1 WITH into_db=$2`, genName, table.TableName)
+			log.Infof(ctx, "Restoring from %s", table.BackupURI)
 			var rows, index, tableBytes int64
 			var discard interface{}
 			res, err := sqlDB.Query(importStmt, table.BackupURI, database)
 			if err != nil {
-				return err
+				return errors.Wrapf(err, "backup: %s", table.BackupURI)
 			}
 			defer res.Close()
 			if !res.Next() {
@@ -619,14 +646,14 @@ func ListFixtures(
 	gensPrefix := config.GCSPrefix + `/`
 	for genIter := b.Objects(ctx, &storage.Query{Prefix: gensPrefix, Delimiter: `/`}); ; {
 		gen, err := genIter.Next()
-		if err == iterator.Done {
+		if errors.Is(err, iterator.Done) {
 			break
 		} else if err != nil {
 			return nil, err
 		}
 		for genConfigIter := b.Objects(ctx, &storage.Query{Prefix: gen.Prefix, Delimiter: `/`}); ; {
 			genConfig, err := genConfigIter.Next()
-			if err == iterator.Done {
+			if errors.Is(err, iterator.Done) {
 				break
 			} else if err != nil {
 				return nil, err

pkg/ccl/workloadccl/fixture_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/stats"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -90,7 +91,7 @@ func TestFixture(t *testing.T) {
 	gcsBucket := os.Getenv(`GS_BUCKET`)
 	gcsKey := os.Getenv(`GS_JSONKEY`)
 	if gcsBucket == "" || gcsKey == "" {
-		t.Skip("GS_BUCKET and GS_JSONKEY env vars must be set")
+		skip.IgnoreLint(t, "GS_BUCKET and GS_JSONKEY env vars must be set")
 	}
 
 	source, err := google.JWTConfigFromJSON([]byte(gcsKey), storage.ScopeReadWrite)

pkg/ccl/workloadccl/format/sstable.go

@@ -24,7 +24,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/workload"
-	"github.com/pkg/errors"
+	"github.com/cockroachdb/errors"
 )
 
 // ToTableDescriptor returns the corresponding TableDescriptor for a workload

pkg/ccl/workloadccl/roachmartccl/roachmart.go

@@ -12,7 +12,6 @@ import (
 	"bytes"
 	"context"
 	gosql "database/sql"
-	"errors"
 	"fmt"
 	"math/rand"
 	"strings"
@@ -21,6 +20,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/cockroach/pkg/workload"
 	"github.com/cockroachdb/cockroach/pkg/workload/histogram"
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/pflag"
 )
 
@@ -200,7 +200,9 @@ func (m *roachmart) Tables() []workload.Table {
 }
 
 // Ops implements the Opser interface.
-func (m *roachmart) Ops(urls []string, reg *histogram.Registry) (workload.QueryLoad, error) {
+func (m *roachmart) Ops(
+	ctx context.Context, urls []string, reg *histogram.Registry,
+) (workload.QueryLoad, error) {
 	sqlDatabase, err := workload.SanitizeUrls(m, m.connFlags.DBOverride, urls)
 	if err != nil {
 		return workload.QueryLoad{}, err

pkg/cli/cli.go

@@ -28,12 +28,13 @@ import (
 	_ "github.com/cockroachdb/cockroach/pkg/workload/bank"       // registers workloads
 	_ "github.com/cockroachdb/cockroach/pkg/workload/bulkingest" // registers workloads
 	workloadcli "github.com/cockroachdb/cockroach/pkg/workload/cli"
-	_ "github.com/cockroachdb/cockroach/pkg/workload/examples" // registers workloads
-	_ "github.com/cockroachdb/cockroach/pkg/workload/kv"       // registers workloads
-	_ "github.com/cockroachdb/cockroach/pkg/workload/movr"     // registers workloads
-	_ "github.com/cockroachdb/cockroach/pkg/workload/tpcc"     // registers workloads
-	_ "github.com/cockroachdb/cockroach/pkg/workload/tpch"     // registers workloads
-	_ "github.com/cockroachdb/cockroach/pkg/workload/ycsb"     // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/examples"     // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/kv"           // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/movr"         // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/schemachange" // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/tpcc"         // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/tpch"         // registers workloads
+	_ "github.com/cockroachdb/cockroach/pkg/workload/ycsb"         // registers workloads
 	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 )

pkg/cli/cli_test.go

@@ -1401,7 +1401,7 @@ Available Commands:
   version           output version information
   debug             debugging commands
   sqlfmt            format SQL statements
-  workload          [experimental] generators for data and query loads
+  workload          generators for data and query loads
   systembench       Run systembench
   help              Help about any command
 

pkg/cli/demo_cluster.go

@@ -604,7 +604,7 @@ func (c *transientCluster) runWorkload(
 
 	// Dummy registry to prove to the Opser.
 	reg := histogram.NewRegistry(time.Duration(100) * time.Millisecond)
-	ops, err := opser.Ops(sqlUrls, reg)
+	ops, err := opser.Ops(ctx, sqlUrls, reg)
 	if err != nil {
 		return errors.Wrap(err, "unable to create workload")
 	}

pkg/cmd/roachprod/cloud/cluster_cloud.go

@@ -19,7 +19,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod/config"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod/vm"
-	"github.com/pkg/errors"
+	"github.com/cockroachdb/errors"
 )
 
 const vmNameFormat = "user-<clusterid>-<nodeid>"