Merge pull request #5692 from planetscale/ss-create-lookup

vrepl: CreateLookupVindex command with backfill

go.mod

@@ -20,7 +20,7 @@ require (
  github.com/evanphx/json-patch v4.5.0+incompatible
  github.com/go-critic/go-critic v0.4.0 // indirect
  github.com/go-ini/ini v1.12.0 // indirect
- github.com/gogo/protobuf v1.3.1 // indirect
+ github.com/gogo/protobuf v1.3.1
  github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
  github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
  github.com/golang/mock v1.3.1

go/vt/vtctl/vtctl.go

@@ -312,8 +312,11 @@ var commands = []commandGroup{
  "[-skip_schema_copy] <keyspace.workflow> <source_shards> <target_shards>",
  "Start a Resharding process. Example: Reshard ks.workflow001 '0' '-80,80-'"},
  {"Migrate", commandMigrate,
- "[-cell=<cell>] [-tablet_types=<tablet_types>] -workflow=<workflow> <source_keyspace> <target_keyspace> <table_specs>",
+ "[-cell=<cell>] [-tablet_types=<source_tablet_types>] -workflow=<workflow> <source_keyspace> <target_keyspace> <table_specs>",
  `Start a table(s) migration, table_specs is a list of tables or the tables section of the vschema for the target keyspace. Example: '{"t1":{"column_vindexes": [{""column": "id1", "name": "hash"}]}, "t2":{"column_vindexes": [{""column": "id2", "name": "hash"}]}}`},
+ {"CreateLookupVindex", commandCreateLookupVindex,
+ "[-cell=<cell>] [-tablet_types=<source_tablet_types>] <keyspace> <json_spec>",
+ `Create and backfill a lookup vindex. the json_spec must contain the vindex and colvindex specs for the new lookup.`},
  {"Materialize", commandMaterialize,
  `<json_spec>, example : '{"workflow": "aaa", "source_keyspace": "source", "target_keyspace": "target", "table_settings": [{"target_table": "customer", "source_expression": "select * from customer", "create_ddl": "copy"}]}'`,
  "Performs materialization based on the json spec."},
@@ -1830,6 +1833,23 @@ func commandMigrate(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.F
  return wr.Migrate(ctx, *workflow, source, target, tableSpecs, *cell, *tabletTypes)
 }
 
+func commandCreateLookupVindex(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
+ cell := subFlags.String("cell", "", "Cell to replicate from.")
+ tabletTypes := subFlags.String("tablet_types", "", "Source tablet types to replicate from.")
+ if err := subFlags.Parse(args); err != nil {
+ return err
+ }
+ if subFlags.NArg() != 2 {
+ return fmt.Errorf("two arguments are required: keyspace and json_spec")
+ }
+ keyspace := subFlags.Arg(0)
+ specs := &vschemapb.Keyspace{}
+ if err := json2.Unmarshal([]byte(subFlags.Arg(1)), specs); err != nil {
+ return err
+ }
+ return wr.CreateLookupVindex(ctx, keyspace, specs, *cell, *tabletTypes)
+}
+
 func commandMaterialize(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
  if err := subFlags.Parse(args); err != nil {
  return err

go/vt/vtgate/vindexes/vschema.go

@@ -25,6 +25,7 @@ import (
  "strings"
 
  "vitess.io/vitess/go/json2"
+ "vitess.io/vitess/go/sqltypes"
  "vitess.io/vitess/go/vt/sqlparser"
 
  querypb "vitess.io/vitess/go/vt/proto/query"
@@ -614,6 +615,19 @@ func LoadFormalKeyspace(filename string) (*vschemapb.Keyspace, error) {
  return formal, nil
 }
 
+// ChooseVindexForType chooses the most appropriate vindex for the give type.
+func ChooseVindexForType(typ querypb.Type) (string, error) {
+ switch {
+ case sqltypes.IsIntegral(typ):
+ return "hash", nil
+ case sqltypes.IsText(typ):
+ return "unicode_loose_md5", nil
+ case sqltypes.IsBinary(typ):
+ return "binary_md5", nil
+ }
+ return "", fmt.Errorf("type %v is not recommended for a vindex", typ)
+}
+
 // FindBestColVindex finds the best ColumnVindex for VReplication.
 func FindBestColVindex(table *Table) (*ColumnVindex, error) {
  if len(table.ColumnVindexes) == 0 {

go/vt/vtgate/vindexes/vschema_test.go

@@ -820,6 +820,115 @@ func TestVSchemaRoutingRules(t *testing.T) {
  }
 }
 
+func TestChooseVindexForType(t *testing.T) {
+ testcases := []struct {
+ in querypb.Type
+ out string
+ }{{
+ in: sqltypes.Null,
+ out: "",
+ }, {
+ in: sqltypes.Int8,
+ out: "hash",
+ }, {
+ in: sqltypes.Uint8,
+ out: "hash",
+ }, {
+ in: sqltypes.Int16,
+ out: "hash",
+ }, {
+ in: sqltypes.Uint16,
+ out: "hash",
+ }, {
+ in: sqltypes.Int24,
+ out: "hash",
+ }, {
+ in: sqltypes.Uint24,
+ out: "hash",
+ }, {
+ in: sqltypes.Int32,
+ out: "hash",
+ }, {
+ in: sqltypes.Uint32,
+ out: "hash",
+ }, {
+ in: sqltypes.Int64,
+ out: "hash",
+ }, {
+ in: sqltypes.Uint64,
+ out: "hash",
+ }, {
+ in: sqltypes.Float32,
+ out: "hash",
+ }, {
+ in: sqltypes.Float64,
+ out: "",
+ }, {
+ in: sqltypes.Timestamp,
+ out: "",
+ }, {
+ in: sqltypes.Date,
+ out: "",
+ }, {
+ in: sqltypes.Time,
+ out: "",
+ }, {
+ in: sqltypes.Datetime,
+ out: "",
+ }, {
+ in: sqltypes.Year,
+ out: "hash",
+ }, {
+ in: sqltypes.Decimal,
+ out: "",
+ }, {
+ in: sqltypes.Text,
+ out: "unicode_loose_md5",
+ }, {
+ in: sqltypes.Blob,
+ out: "binary_md5",
+ }, {
+ in: sqltypes.VarChar,
+ out: "unicode_loose_md5",
+ }, {
+ in: sqltypes.VarBinary,
+ out: "binary_md5",
+ }, {
+ in: sqltypes.Char,
+ out: "unicode_loose_md5",
+ }, {
+ in: sqltypes.Binary,
+ out: "binary_md5",
+ }, {
+ in: sqltypes.Bit,
+ out: "",
+ }, {
+ in: sqltypes.Enum,
+ out: "",
+ }, {
+ in: sqltypes.Set,
+ out: "",
+ }, {
+ in: sqltypes.Geometry,
+ out: "",
+ }, {
+ in: sqltypes.TypeJSON,
+ out: "",
+ }, {
+ in: sqltypes.Expression,
+ out: "",
+ }}
+
+ for _, tcase := range testcases {
+ out, err := ChooseVindexForType(tcase.in)
+ if out == "" {
+ assert.Error(t, err, tcase.in)
+ continue
+ }
+ assert.Equal(t, out, tcase.out, tcase.in)
+ }
+}
+
 func TestFindBestColVindex(t *testing.T) {
  testSrvVSchema := &vschemapb.SrvVSchema{
  Keyspaces: map[string]*vschemapb.Keyspace{