CherryPick(vitessio#13458): vtgate v3 planner removal (vitessio#2674) (…

…vitessio#2680)

* cherry pick of 13458

* fix boost build



---------

Signed-off-by: Harshit Gangal <harshit@planetscale.com>
Co-authored-by: Harshit Gangal <harshit@planetscale.com>

changelog/18.0/18.0.0/summary.md

@@ -8,6 +8,7 @@
  - [VTOrc flag `--allow-emergency-reparent`](#new-flag-toggle-ers)
  - **[Deprecations and Deletions](#deprecations-and-deletions)**
  - [Deprecated Flags](#deprecated-flags)
+ - [Deleted `V3` planner](#deleted-v3)
  - [Deleted `k8stopo`](#deleted-k8stopo)
  - [Deleted `vtgr`](#deleted-vtgr)
  - **[New stats](#new-stats)**
@@ -49,6 +50,10 @@ Throttler related `vttablet` flags:
 - `--throttle_check_as_check_self` is deprecated and will be removed in `v19.0`
 - `--throttler-config-via-topo` is deprecated after asummed `true` in `v17.0`. It will be removed in a future version.
 
+#### <a id="deleted-v3"/>Deleted `v3` planner
+
+The `Gen4` planner has been the default planner since Vitess 14. The `v3` planner was deprecated in Vitess 15 and has now been removed in this release.
+
 #### <a id="deleted-k8stopo"/>Deleted `k8stopo`
 
 The `k8stopo` has been deprecated in Vitess 17, also see https://github.com/vitessio/vitess/issues/13298. With Vitess 18

go/cmd/vtcombo/main.go

@@ -61,7 +61,7 @@ var (
  mysqlPort = flags.Int("mysql_port", 3306, "mysql port")
  externalTopoServer = flags.Bool("external_topo_server", false, "Should vtcombo use an external topology server instead of starting its own in-memory topology server. "+
  "If true, vtcombo will use the flags defined in topo/server.go to open topo server")
- plannerName = flags.String("planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.")
+ plannerName = flags.String("planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right")
 
  tpb vttestpb.VTTestTopology
  ts *topo.Server

go/cmd/vtexplain/vtexplain.go

@@ -63,7 +63,7 @@ func registerFlags(fs *pflag.FlagSet) {
  fs.StringVar(&replicationMode, "replication-mode", replicationMode, "The replication mode to simulate -- must be set to either ROW or STATEMENT")
  fs.BoolVar(&normalize, "normalize", normalize, "Whether to enable vtgate normalization")
  fs.StringVar(&dbName, "dbname", dbName, "Optional database target to override normal routing")
- fs.StringVar(&plannerVersionStr, "planner-version", plannerVersionStr, "Sets the query planner version to use when generating the explain output. Valid values are V3 and Gen4. An empty value will use VTGate's default planner")
+ fs.StringVar(&plannerVersionStr, "planner-version", plannerVersionStr, "Sets the default planner to use. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right")
  fs.IntVar(&numShards, "shards", numShards, "Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored.")
  fs.StringVar(&executionMode, "execution-mode", executionMode, "The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc")
  fs.StringVar(&outputMode, "output-mode", outputMode, "Output in human-friendly text or json")
@@ -113,8 +113,8 @@ func main() {
 
 func parseAndRun() error {
  plannerVersion, _ := plancontext.PlannerNameToVersion(plannerVersionStr)
- if plannerVersionStr != "" && plannerVersion != querypb.ExecuteOptions_V3 && plannerVersion != querypb.ExecuteOptions_Gen4 {
- return fmt.Errorf("invalid value specified for planner-version of '%s' -- valid values are V3 and Gen4 or an empty value to use the default planner", plannerVersionStr)
+ if plannerVersionStr != "" && plannerVersion != querypb.ExecuteOptions_Gen4 {
+ return fmt.Errorf("invalid value specified for planner-version of '%s' -- valid value is Gen4 or an empty value to use the default planner", plannerVersionStr)
  }
 
  sql, err := getFileParam(sqlFlag, sqlFileFlag, "sql", true)

go/cmd/vtgate/vtgate.go

@@ -48,7 +48,7 @@ var (
 func registerFlags(fs *pflag.FlagSet) {
  fs.StringVar(&cell, "cell", cell, "cell to use")
  fs.Var((*topoproto.TabletTypeListFlag)(&tabletTypesToWait), "tablet_types_to_wait", "Wait till connected for specified tablet types during Gateway initialization. Should be provided as a comma-separated set of tablet types.")
- fs.StringVar(&plannerName, "planner-version", plannerName, "Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.")
+ fs.StringVar(&plannerName, "planner-version", plannerName, "Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right")
 
  acl.RegisterFlags(fs)
 }

go/cmd/vttestserver/main.go

@@ -141,7 +141,7 @@ func registerFlags(fs *pflag.FlagSet) {
 
  fs.StringVar(&config.Charset, "charset", "utf8mb4", "MySQL charset")
 
- fs.StringVar(&config.PlannerVersion, "planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails.")
+ fs.StringVar(&config.PlannerVersion, "planner-version", "", "Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right")
 
  fs.StringVar(&config.SnapshotFile, "snapshot_file", "",
  "A MySQL DB snapshot file")

go/flags/endtoend/vtexplain.txt

@@ -44,7 +44,7 @@ Usage of vtexplain:
  --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess")
  --normalize Whether to enable vtgate normalization
  --output-mode string Output in human-friendly text or json (default "text")
- --planner-version string Sets the query planner version to use when generating the explain output. Valid values are V3 and Gen4. An empty value will use VTGate's default planner
+ --planner-version string Sets the default planner to use. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right
  --pprof strings enable profiling
  --purge_logs_interval duration how often try to remove old logs (default 1h0m0s)
  --replication-mode string The replication mode to simulate -- must be set to either ROW or STATEMENT (default "ROW")

go/flags/endtoend/vtgate.txt

@@ -155,7 +155,7 @@ Usage of vtgate:
  --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s)
  --opentsdb_uri string URI of opentsdb /api/put method
  --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown.
- --planner-version string Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the gen4 planner and falls back to the V3 planner if the gen4 fails.
+ --planner-version string Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right
  --port int port for the server
  --pprof strings enable profiling
  --proxy_protocol Enable HAProxy PROXY protocol on MySQL listener socket

go/flags/endtoend/vttestserver.txt

@@ -91,7 +91,7 @@ Usage of vttestserver:
  --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s)
  --persistent_mode If this flag is set, the MySQL data directory is not cleaned up when LocalCluster.TearDown() is called. This is useful for running vttestserver as a database container in local developer environments. Note that db migration files (--schema_dir option) and seeding of random data (--initialize_with_random_data option) will only run during cluster startup if the data directory does not already exist. vschema migrations are run every time the cluster starts, since persistence for the topology server has not been implemented yet
  --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown.
- --planner-version string Sets the default planner to use when the session has not changed it. Valid values are: V3, V3Insert, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails.
+ --planner-version string Sets the default planner to use when the session has not changed it. Valid values are: Gen4, Gen4Greedy, Gen4Left2Right
  --pool_hostname_resolve_interval duration if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled)
  --port int Port to use for vtcombo. If this is 0, a random port will be chosen.
  --pprof strings enable profiling

go/test/endtoend/cluster/vtgate_process.go

@@ -65,7 +65,7 @@ type VtgateProcess struct {
  exit chan error
 }
 
-const defaultVtGatePlannerVersion = planbuilder.Gen4CompareV3
+const defaultVtGatePlannerVersion = planbuilder.Gen4
 
 // Setup starts Vtgate process with required arguements
 func (vtgate *VtgateProcess) Setup() (err error) {

go/test/endtoend/vtgate/queries/misc/misc_test.go

@@ -151,12 +151,12 @@ func TestQueryTimeoutWithTables(t *testing.T) {
  }
  // too much data added in the loop, do drop and recreate the table.
  defer func() {
- mcmp.Exec("drop table t1")
+ mcmp.Exec("drop /*vt+ QUERY_TIMEOUT_MS=1000 */ table t1")
  mcmp.Exec(schemaSQL)
  }()
 
- utils.Exec(t, mcmp.VtConn, "select count(*) from t1 where id1 > 31")
- utils.Exec(t, mcmp.VtConn, "select /*vt+ PLANNER=gen4 QUERY_TIMEOUT_MS=100 */ count(*) from t1 where id1 > 31")
+ utils.Exec(t, mcmp.VtConn, "select /*vt+ PLANNER=gen4 QUERY_TIMEOUT_MS=1000 */ count(*) from t1 where id1 > 31")
+ utils.Exec(t, mcmp.VtConn, "select /*vt+ PLANNER=gen4 QUERY_TIMEOUT_MS=1000 */ count(*) from t1 where id1 > 31")
 
  // the query usually takes more than 5ms to return. So this should fail.
  _, err = utils.ExecAllowError(t, mcmp.VtConn, "select /*vt+ PLANNER=gen4 QUERY_TIMEOUT_MS=1 */ count(*) from t1 where id1 > 31")

go/test/endtoend/vtgate/queries/misc/schema.sql

@@ -1,4 +1,4 @@
-create table if not exists t1(
+create /*vt+ QUERY_TIMEOUT_MS=1000 */ table if not exists t1(
  id1 bigint,
  id2 bigint,
  primary key(id1)

go/test/endtoend/vtgate/reservedconn/reconnect1/main_test.go

@@ -179,9 +179,17 @@ func TestServingChangeStreaming(t *testing.T) {
  rdonlyTablet.Type = "replica"
 
  // this should fail as there is no rdonly present
+ // This can also close the streaming connection if it goes to 80- shard first and sends the fields from that.
+ // Current, stream logic is to close the server connection if partial stream result is sent and an error is received later.
  _, err = utils.ExecAllowError(t, conn, "select * from test")
  require.Error(t, err)
 
+ // check if connection is still available
+ _, err = utils.ExecAllowError(t, conn, "select 1")
+ if err != nil {
+ t.Skip("connection is closed, cannot continue with the test")
+ }
+
  // changing replica tablet to rdonly to make rdonly available for serving.
  replicaTablet := clusterInstance.Keyspaces[0].Shards[0].Replica()
  err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", replicaTablet.Alias, "rdonly")

go/vt/sqlparser/ast.go

@@ -2289,11 +2289,6 @@ type (
 
  // ColName represents a column name.
  ColName struct {
- // Metadata is not populated by the parser.
- // It's a placeholder for analyzers to store
- // additional data, typically info about which
- // table or column this node references.
- Metadata any
  Name IdentifierCI
  Qualifier TableName
  }

go/vt/sqlparser/predicate_rewriting.go

@@ -43,10 +43,6 @@ func RewritePredicate(ast SQLNode) SQLNode {
  exprChanged = true
  cursor.Replace(rewritten)
  }
-
- if col, isCol := cursor.node.(*ColName); isCol {
- col.Metadata = nil
- }
  return !exprChanged
  })
 

go/vt/vtgate/engine/boost_compare.go

@@ -2,6 +2,7 @@ package engine
 
 import (
  "context"
+ "encoding/json"
  "sync"
 
  "vitess.io/vitess/go/sqltypes"
@@ -181,3 +182,50 @@ func (b *BoostCompare) compareResults(result *sqltypes.Result, gen4Res *sqltypes
  }
  return nil
 }
+
+func printMismatch(leftResult, rightResult *sqltypes.Result, leftPrimitive, rightPrimitive Primitive, leftName, rightName string) {
+ log.Errorf("Results of %s and %s are not equal. Displaying diff.", rightName, leftName)
+
+ // get right plan and print it
+ rightplan := &Plan{
+ Instructions: rightPrimitive,
+ }
+ rightJSON, _ := json.MarshalIndent(rightplan, "", " ")
+ log.Errorf("%s's plan:\n%s", rightName, string(rightJSON))
+
+ // get left's plan and print it
+ leftplan := &Plan{
+ Instructions: leftPrimitive,
+ }
+ leftJSON, _ := json.MarshalIndent(leftplan, "", " ")
+ log.Errorf("%s's plan:\n%s", leftName, string(leftJSON))
+
+ log.Errorf("%s's results:\n", rightName)
+ log.Errorf("\t[rows affected: %d]\n", rightResult.RowsAffected)
+ for _, row := range rightResult.Rows {
+ log.Errorf("\t%s", row)
+ }
+ log.Errorf("%s's results:\n", leftName)
+ log.Errorf("\t[rows affected: %d]\n", leftResult.RowsAffected)
+ for _, row := range leftResult.Rows {
+ log.Errorf("\t%s", row)
+ }
+ log.Error("End of diff.")
+}
+
+// CompareErrors compares the two errors, and if they don't match, produces an error
+func CompareErrors(leftErr, rightErr error, leftName, rightName string) error {
+ if leftErr != nil && rightErr != nil {
+ if leftErr.Error() == rightErr.Error() {
+ return rightErr
+ }
+ return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "%s and %s failed with different errors: %s: [%s], %s: [%s]", leftName, rightName, leftErr.Error(), rightErr.Error(), leftName, rightName)
+ }
+ if leftErr == nil && rightErr != nil {
+ return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "%s failed while %s did not: %s", rightName, rightErr.Error(), leftName)
+ }
+ if leftErr != nil && rightErr == nil {
+ return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "%s failed while %s did not: %s", leftName, leftErr.Error(), rightName)
+ }
+ return nil
+}