From 7827fff5e8f18bcce6a61a227d96ca95be09e1f7 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 5 Jun 2021 16:38:23 -0400 Subject: [PATCH 1/4] [vtctldserver] wip -- start tracing all vtctldserver endpoints Signed-off-by: Andrew Mason --- go/vt/topo/topoproto/keyspace.go | 22 ++++++++ go/vt/vtctl/grpcvtctldserver/server.go | 73 ++++++++++++++++++++++++++ go/vt/vtctl/grpcvtctldserver/topo.go | 9 ++++ 3 files changed, 104 insertions(+) diff --git a/go/vt/topo/topoproto/keyspace.go b/go/vt/topo/topoproto/keyspace.go index 625ec0b7293..d40c8698827 100644 --- a/go/vt/topo/topoproto/keyspace.go +++ b/go/vt/topo/topoproto/keyspace.go @@ -42,3 +42,25 @@ func KeyspaceTypeString(kt topodatapb.KeyspaceType) string { return str } + +// KeyspaceTypeLString returns the lowercased string representation of a +// KeyspaceType. +func KeyspaceTypeLString(kt topodatapb.KeyspaceType) string { + return strings.ToLower(KeyspaceTypeString(kt)) +} + +// KeyspaceIDTypeString returns the string representation of a KeyspaceIdType. +func KeyspaceIDTypeString(kidType topodatapb.KeyspaceIdType) string { + str, ok := topodatapb.KeyspaceIdType_name[int32(kidType)] + if !ok { + return "UNKNOWN" + } + + return str +} + +// KeyspaceIDTypeLString returns the lowercased string representation of a +// KeyspaceIdType. +func KeyspaceIDTypeLString(kidType topodatapb.KeyspaceIdType) string { + return strings.ToLower(KeyspaceIDTypeString(kidType)) +} diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 55beb6a9b17..40ef1997fd5 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "sync" "time" @@ -84,10 +85,17 @@ func NewVtctldServer(ts *topo.Server) *VtctldServer { // AddCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) AddCellInfo(ctx context.Context, req *vtctldatapb.AddCellInfoRequest) (*vtctldatapb.AddCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.AddCellInfo") + defer span.Finish() + if req.CellInfo.Root == "" { return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "CellInfo.Root must be non-empty") } + span.Annotate("cell", req.Name) + span.Annotate("cell_root", req.CellInfo.Root) + span.Annotate("cell_address", req.CellInfo.ServerAddress) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -100,6 +108,12 @@ func (s *VtctldServer) AddCellInfo(ctx context.Context, req *vtctldatapb.AddCell // AddCellsAlias is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) AddCellsAlias(ctx context.Context, req *vtctldatapb.AddCellsAliasRequest) (*vtctldatapb.AddCellsAliasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.AddCellsAlias") + defer span.Finish() + + span.Annotate("cells_alias", req.Name) + span.Annotate("cells", strings.Join(req.Cells, ",")) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -112,6 +126,12 @@ func (s *VtctldServer) AddCellsAlias(ctx context.Context, req *vtctldatapb.AddCe // ApplyRoutingRules is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) ApplyRoutingRules(ctx context.Context, req *vtctldatapb.ApplyRoutingRulesRequest) (*vtctldatapb.ApplyRoutingRulesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ApplyRoutingRules") + defer span.Finish() + + span.Annotate("skip_rebuild", req.SkipRebuild) + span.Annotate("rebuild_cells", strings.Join(req.RebuildCells, ",")) + if err := s.ts.SaveRoutingRules(ctx, req.RoutingRules); err != nil { return nil, err } @@ -191,6 +211,13 @@ func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyV // ChangeTabletType is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) ChangeTabletType(ctx context.Context, req *vtctldatapb.ChangeTabletTypeRequest) (*vtctldatapb.ChangeTabletTypeResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ChangeTabletType") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.TabletAlias)) + span.Annotate("dry_run", req.DryRun) + span.Annotate("tablet_type", topoproto.TabletTypeLString(req.DbType)) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -199,6 +226,8 @@ func (s *VtctldServer) ChangeTabletType(ctx context.Context, req *vtctldatapb.Ch return nil, err } + span.Annotate("before_tablet_type", topoproto.TabletTypeLString(tablet.Type)) + if !topo.IsTrivialTypeChange(tablet.Type, req.DbType) { return nil, fmt.Errorf("tablet %v type change %v -> %v is not an allowed transition for ChangeTabletType", req.TabletAlias, tablet.Type, req.DbType) } @@ -237,6 +266,16 @@ func (s *VtctldServer) ChangeTabletType(ctx context.Context, req *vtctldatapb.Ch // CreateKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) CreateKeyspace(ctx context.Context, req *vtctldatapb.CreateKeyspaceRequest) (*vtctldatapb.CreateKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.CreateKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Name) + span.Annotate("keyspace_type", topoproto.KeyspaceTypeLString(req.Type)) + span.Annotate("sharding_column_name", req.ShardingColumnName) + span.Annotate("sharding_column_type", topoproto.KeyspaceIDTypeLString(req.ShardingColumnType)) + span.Annotate("force", req.Force) + span.Annotate("allow_empty_vschema", req.AllowEmptyVSchema) + switch req.Type { case topodatapb.KeyspaceType_NORMAL: case topodatapb.KeyspaceType_SNAPSHOT: @@ -247,6 +286,9 @@ func (s *VtctldServer) CreateKeyspace(ctx context.Context, req *vtctldatapb.Crea if req.SnapshotTime == nil { return nil, errors.New("SnapshotTime is required for SNAPSHOT keyspaces") } + + span.Annotate("base_keyspace", req.BaseKeyspace) + span.Annotate("snapshot_time", req.SnapshotTime) // TODO: get a proper string repr default: return nil, fmt.Errorf("unknown keyspace type %v", req.Type) } @@ -325,6 +367,14 @@ func (s *VtctldServer) CreateKeyspace(ctx context.Context, req *vtctldatapb.Crea // CreateShard is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) CreateShard(ctx context.Context, req *vtctldatapb.CreateShardRequest) (*vtctldatapb.CreateShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.CreateShard") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.ShardName) + span.Annotate("force", req.Force) + span.Annotate("include_parent", req.IncludeParent) + if req.IncludeParent { log.Infof("Creating empty keyspace for %s", req.Keyspace) if err := s.ts.CreateKeyspace(ctx, req.Keyspace, &topodatapb.Keyspace{}); err != nil { @@ -376,6 +426,12 @@ func (s *VtctldServer) CreateShard(ctx context.Context, req *vtctldatapb.CreateS // DeleteCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteCellInfo(ctx context.Context, req *vtctldatapb.DeleteCellInfoRequest) (*vtctldatapb.DeleteCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteCellInfo") + defer span.Finish() + + span.Annotate("cell", req.Name) + span.Annotate("force", req.Force) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -388,6 +444,11 @@ func (s *VtctldServer) DeleteCellInfo(ctx context.Context, req *vtctldatapb.Dele // DeleteCellsAlias is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteCellsAlias(ctx context.Context, req *vtctldatapb.DeleteCellsAliasRequest) (*vtctldatapb.DeleteCellsAliasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteCellsAlias") + defer span.Finish() + + span.Annotate("cells_alias", req.Name) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -400,6 +461,12 @@ func (s *VtctldServer) DeleteCellsAlias(ctx context.Context, req *vtctldatapb.De // DeleteKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteKeyspace(ctx context.Context, req *vtctldatapb.DeleteKeyspaceRequest) (*vtctldatapb.DeleteKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("recursive", req.Recursive) + shards, err := s.ts.GetShardNames(ctx, req.Keyspace) if err != nil { return nil, err @@ -446,6 +513,9 @@ func (s *VtctldServer) DeleteKeyspace(ctx context.Context, req *vtctldatapb.Dele // DeleteShards is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteShards(ctx context.Context, req *vtctldatapb.DeleteShardsRequest) (*vtctldatapb.DeleteShardsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteShards") + defer span.Finish() + for _, shard := range req.Shards { if err := deleteShard(ctx, s.ts, shard.Keyspace, shard.Name, req.Recursive, req.EvenIfServing); err != nil { return nil, err @@ -457,6 +527,9 @@ func (s *VtctldServer) DeleteShards(ctx context.Context, req *vtctldatapb.Delete // DeleteTablets is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteTablets(ctx context.Context, req *vtctldatapb.DeleteTabletsRequest) (*vtctldatapb.DeleteTabletsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteTablets") + defer span.Finish() + for _, alias := range req.TabletAliases { if err := deleteTablet(ctx, s.ts, alias, req.AllowPrimary); err != nil { return nil, err diff --git a/go/vt/vtctl/grpcvtctldserver/topo.go b/go/vt/vtctl/grpcvtctldserver/topo.go index 7de161bcd22..14c6944182d 100644 --- a/go/vt/vtctl/grpcvtctldserver/topo.go +++ b/go/vt/vtctl/grpcvtctldserver/topo.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -31,6 +32,14 @@ import ( ) func deleteShard(ctx context.Context, ts *topo.Server, keyspace string, shard string, recursive bool, evenIfServing bool) error { + span, ctx := trace.NewSpan(ctx, "VtctldServer.deleteShard") + defer span.Finish() + + span.Annotate("keyspace", keyspace) + span.Annotate("shard", shard) + span.Annotate("recursive", recursive) + span.Annotate("even_if_serving", evenIfServing) + // Read the Shard object. If it's not in the topo, try to clean up the topo // anyway. shardInfo, err := ts.GetShard(ctx, keyspace, shard) From f649cb696eef8f3cd080646132af15f8ad478ba2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 6 Jun 2021 13:33:03 -0400 Subject: [PATCH 2/4] Fix typo in comment Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/topo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/topo.go b/go/vt/vtctl/grpcvtctldserver/topo.go index 14c6944182d..4363a0c2174 100644 --- a/go/vt/vtctl/grpcvtctldserver/topo.go +++ b/go/vt/vtctl/grpcvtctldserver/topo.go @@ -243,7 +243,7 @@ func removeShardCell(ctx context.Context, ts *topo.Server, cell string, keyspace if recursive { log.Infof("Deleting all tablets in cell %v in shard %v/%v", cell, keyspace, shardName) for _, node := range replication.Nodes { - // We don't care about scraping our updating the replication + // We don't care about scrapping or updating the replication // graph, because we're about to delete the entire replication // graph. log.Infof("Deleting tablet %v", topoproto.TabletAliasString(node.TabletAlias)) From 100e294b6314b8be8b84b744c33dc5729c65174a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 6 Jun 2021 13:37:18 -0400 Subject: [PATCH 3/4] fixup! [vtctldserver] wip -- start tracing all vtctldserver endpoints Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/server.go | 195 ++++++++++++++++++++++++- go/vt/vtctl/grpcvtctldserver/topo.go | 25 ++++ 2 files changed, 217 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 40ef1997fd5..122cc1dceca 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -516,6 +516,10 @@ func (s *VtctldServer) DeleteShards(ctx context.Context, req *vtctldatapb.Delete span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteShards") defer span.Finish() + span.Annotate("num_shards", len(req.Shards)) + span.Annotate("even_if_serving", req.EvenIfServing) + span.Annotate("recursive", req.Recursive) + for _, shard := range req.Shards { if err := deleteShard(ctx, s.ts, shard.Keyspace, shard.Name, req.Recursive, req.EvenIfServing); err != nil { return nil, err @@ -530,6 +534,9 @@ func (s *VtctldServer) DeleteTablets(ctx context.Context, req *vtctldatapb.Delet span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteTablets") defer span.Finish() + span.Annotate("num_tablets", len(req.TabletAliases)) + span.Annotate("allow_primary", req.AllowPrimary) + for _, alias := range req.TabletAliases { if err := deleteTablet(ctx, s.ts, alias, req.AllowPrimary); err != nil { return nil, err @@ -541,6 +548,16 @@ func (s *VtctldServer) DeleteTablets(ctx context.Context, req *vtctldatapb.Delet // EmergencyReparentShard is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldatapb.EmergencyReparentShardRequest) (*vtctldatapb.EmergencyReparentShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.EmergencyReparentShard") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + span.Annotate("new_primary_alias", topoproto.TabletAliasString(req.NewPrimary)) + + ignoreReplicaAliases := topoproto.TabletAliasList(req.IgnoreReplicas).ToStringSlice() + span.Annotate("ignore_replicas", strings.Join(ignoreReplicaAliases, ",")) + waitReplicasTimeout, ok, err := protoutil.DurationFromProto(req.WaitReplicasTimeout) if err != nil { return nil, err @@ -548,6 +565,8 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat waitReplicasTimeout = time.Second * 30 } + span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + m := sync.RWMutex{} logstream := []*logutilpb.Event{} logger := logutil.NewCallbackLogger(func(e *logutilpb.Event) { @@ -562,7 +581,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat req.Shard, reparentutil.EmergencyReparentOptions{ NewPrimaryAlias: req.NewPrimary, - IgnoreReplicas: sets.NewString(topoproto.TabletAliasList(req.IgnoreReplicas).ToStringSlice()...), + IgnoreReplicas: sets.NewString(ignoreReplicaAliases...), WaitReplicasTimeout: waitReplicasTimeout, }, ) @@ -592,6 +611,11 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat // FindAllShardsInKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) FindAllShardsInKeyspace(ctx context.Context, req *vtctldatapb.FindAllShardsInKeyspaceRequest) (*vtctldatapb.FindAllShardsInKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.FindAllShardsInKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + result, err := s.ts.FindAllShardsInKeyspace(ctx, req.Keyspace) if err != nil { return nil, err @@ -613,14 +637,21 @@ func (s *VtctldServer) FindAllShardsInKeyspace(ctx context.Context, req *vtctlda // GetBackups is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) GetBackups(ctx context.Context, req *vtctldatapb.GetBackupsRequest) (*vtctldatapb.GetBackupsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetBackups") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + bs, err := backupstorage.GetBackupStorage() if err != nil { return nil, err } - defer bs.Close() bucket := filepath.Join(req.Keyspace, req.Shard) + span.Annotate("backup_path", bucket) + bhs, err := bs.ListBackups(ctx, bucket) if err != nil { return nil, err @@ -639,6 +670,9 @@ func (s *VtctldServer) GetBackups(ctx context.Context, req *vtctldatapb.GetBacku // GetCellInfoNames is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetCellInfoNames(ctx context.Context, req *vtctldatapb.GetCellInfoNamesRequest) (*vtctldatapb.GetCellInfoNamesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetCellInfoNames") + defer span.Finish() + names, err := s.ts.GetCellInfoNames(ctx) if err != nil { return nil, err @@ -649,10 +683,15 @@ func (s *VtctldServer) GetCellInfoNames(ctx context.Context, req *vtctldatapb.Ge // GetCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetCellInfo(ctx context.Context, req *vtctldatapb.GetCellInfoRequest) (*vtctldatapb.GetCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetCellInfo") + defer span.Finish() + if req.Cell == "" { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell field is required") } + span.Annotate("cell", req.Cell) + // We use a strong read, because users using this command want the latest // data, and this is user-generated, not used in any automated process. strongRead := true @@ -666,6 +705,9 @@ func (s *VtctldServer) GetCellInfo(ctx context.Context, req *vtctldatapb.GetCell // GetCellsAliases is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetCellsAliases(ctx context.Context, req *vtctldatapb.GetCellsAliasesRequest) (*vtctldatapb.GetCellsAliasesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetCellsAliases") + defer span.Finish() + strongRead := true aliases, err := s.ts.GetCellsAliases(ctx, strongRead) if err != nil { @@ -677,6 +719,11 @@ func (s *VtctldServer) GetCellsAliases(ctx context.Context, req *vtctldatapb.Get // GetKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetKeyspace(ctx context.Context, req *vtctldatapb.GetKeyspaceRequest) (*vtctldatapb.GetKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + keyspace, err := s.ts.GetKeyspace(ctx, req.Keyspace) if err != nil { return nil, err @@ -692,6 +739,9 @@ func (s *VtctldServer) GetKeyspace(ctx context.Context, req *vtctldatapb.GetKeys // GetKeyspaces is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetKeyspaces(ctx context.Context, req *vtctldatapb.GetKeyspacesRequest) (*vtctldatapb.GetKeyspacesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetKeyspaces") + defer span.Finish() + names, err := s.ts.GetKeyspaces(ctx) if err != nil { return nil, err @@ -713,6 +763,9 @@ func (s *VtctldServer) GetKeyspaces(ctx context.Context, req *vtctldatapb.GetKey // GetRoutingRules is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetRoutingRules(ctx context.Context, req *vtctldatapb.GetRoutingRulesRequest) (*vtctldatapb.GetRoutingRulesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetRoutingRules") + defer span.Finish() + rr, err := s.ts.GetRoutingRules(ctx) if err != nil { return nil, err @@ -725,11 +778,22 @@ func (s *VtctldServer) GetRoutingRules(ctx context.Context, req *vtctldatapb.Get // GetSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSchema(ctx context.Context, req *vtctldatapb.GetSchemaRequest) (*vtctldatapb.GetSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSchema") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.TabletAlias)) + tablet, err := s.ts.GetTablet(ctx, req.TabletAlias) if err != nil { return nil, fmt.Errorf("GetTablet(%v) failed: %w", req.TabletAlias, err) } + span.Annotate("tables", strings.Join(req.Tables, ",")) + span.Annotate("exclude_tables", strings.Join(req.ExcludeTables, ",")) + span.Annotate("include_views", req.IncludeViews) + span.Annotate("table_names_only", req.TableNamesOnly) + span.Annotate("table_sizes_only", req.TableSizesOnly) + sd, err := s.tmc.GetSchema(ctx, tablet.Tablet, req.Tables, req.ExcludeTables, req.IncludeViews) if err != nil { return nil, fmt.Errorf("GetSchema(%v, %v, %v, %v) failed: %w", tablet.Tablet, req.Tables, req.ExcludeTables, req.IncludeViews, err) @@ -767,6 +831,12 @@ func (s *VtctldServer) GetSchema(ctx context.Context, req *vtctldatapb.GetSchema // GetShard is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetShard(ctx context.Context, req *vtctldatapb.GetShardRequest) (*vtctldatapb.GetShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetShard") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.ShardName) + shard, err := s.ts.GetShard(ctx, req.Keyspace, req.ShardName) if err != nil { return nil, err @@ -783,6 +853,9 @@ func (s *VtctldServer) GetShard(ctx context.Context, req *vtctldatapb.GetShardRe // GetSrvKeyspaces is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSrvKeyspaces(ctx context.Context, req *vtctldatapb.GetSrvKeyspacesRequest) (*vtctldatapb.GetSrvKeyspacesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSrvKeyspaces") + defer span.Finish() + cells := req.Cells if len(cells) == 0 { @@ -794,6 +867,8 @@ func (s *VtctldServer) GetSrvKeyspaces(ctx context.Context, req *vtctldatapb.Get } } + span.Annotate("cells", strings.Join(cells, ",")) + srvKeyspaces := make(map[string]*topodatapb.SrvKeyspace, len(cells)) for _, cell := range cells { @@ -819,6 +894,11 @@ func (s *VtctldServer) GetSrvKeyspaces(ctx context.Context, req *vtctldatapb.Get // GetSrvVSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSrvVSchema(ctx context.Context, req *vtctldatapb.GetSrvVSchemaRequest) (*vtctldatapb.GetSrvVSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSrvVSchema") + defer span.Finish() + + span.Annotate("cell", req.Cell) + vschema, err := s.ts.GetSrvVSchema(ctx, req.Cell) if err != nil { return nil, err @@ -831,6 +911,9 @@ func (s *VtctldServer) GetSrvVSchema(ctx context.Context, req *vtctldatapb.GetSr // GetSrvVSchemas is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSrvVSchemas(ctx context.Context, req *vtctldatapb.GetSrvVSchemasRequest) (*vtctldatapb.GetSrvVSchemasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSrvVSchemas") + defer span.Finish() + allCells, err := s.ts.GetCellInfoNames(ctx) if err != nil { return nil, err @@ -846,6 +929,7 @@ func (s *VtctldServer) GetSrvVSchemas(ctx context.Context, req *vtctldatapb.GetS cells = s1.Intersection(s2).List() } + span.Annotate("cells", strings.Join(cells, ",")) svs := make(map[string]*vschemapb.SrvVSchema, len(cells)) for _, cell := range cells { @@ -870,6 +954,11 @@ func (s *VtctldServer) GetSrvVSchemas(ctx context.Context, req *vtctldatapb.GetS // GetTablet is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetTablet(ctx context.Context, req *vtctldatapb.GetTabletRequest) (*vtctldatapb.GetTabletResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetTablet") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.TabletAlias)) + ti, err := s.ts.GetTablet(ctx, req.TabletAlias) if err != nil { return nil, err @@ -882,6 +971,12 @@ func (s *VtctldServer) GetTablet(ctx context.Context, req *vtctldatapb.GetTablet // GetTablets is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetTablets(ctx context.Context, req *vtctldatapb.GetTabletsRequest) (*vtctldatapb.GetTabletsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetTablets") + defer span.Finish() + + span.Annotate("cells", strings.Join(req.Cells, ",")) + span.Annotate("strict", req.Strict) + // It is possible that an old primary has not yet updated its type in the // topo. In that case, report its type as UNKNOWN. It used to be MASTER but // is no longer the serving primary. @@ -906,11 +1001,16 @@ func (s *VtctldServer) GetTablets(ctx context.Context, req *vtctldatapb.GetTable switch { case len(req.TabletAliases) > 0: + span.Annotate("tablet_aliases", strings.Join(topoproto.TabletAliasList(req.TabletAliases).ToStringSlice(), ",")) + tabletMap, err = s.ts.GetTabletMap(ctx, req.TabletAliases) if err != nil { err = fmt.Errorf("GetTabletMap(%v) failed: %w", req.TabletAliases, err) } case req.Keyspace != "" && req.Shard != "": + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + tabletMap, err = s.ts.GetTabletMapForShard(ctx, req.Keyspace, req.Shard) if err != nil { err = fmt.Errorf("GetTabletMapForShard(%s, %s) failed: %w", req.Keyspace, req.Shard, err) @@ -1043,6 +1143,11 @@ func (s *VtctldServer) GetTablets(ctx context.Context, req *vtctldatapb.GetTable // GetVSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetVSchema(ctx context.Context, req *vtctldatapb.GetVSchemaRequest) (*vtctldatapb.GetVSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetVSchema") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + vschema, err := s.ts.GetVSchema(ctx, req.Keyspace) if err != nil { return nil, err @@ -1066,6 +1171,9 @@ func (s *VtctldServer) GetWorkflows(ctx context.Context, req *vtctldatapb.GetWor // InitShardPrimary is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) InitShardPrimary(ctx context.Context, req *vtctldatapb.InitShardPrimaryRequest) (*vtctldatapb.InitShardPrimaryResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.InitShardPrimary") + defer span.Finish() + if req.Keyspace == "" { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "keyspace field is required") } @@ -1081,6 +1189,11 @@ func (s *VtctldServer) InitShardPrimary(ctx context.Context, req *vtctldatapb.In waitReplicasTimeout = time.Second * 30 } + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + span.Annotate("force", req.Force) + ctx, unlock, err := s.ts.LockShard(ctx, req.Keyspace, req.Shard, fmt.Sprintf("InitShardPrimary(%v)", topoproto.TabletAliasString(req.PrimaryElectTabletAlias))) if err != nil { return nil, err @@ -1092,7 +1205,7 @@ func (s *VtctldServer) InitShardPrimary(ctx context.Context, req *vtctldatapb.In logstream := []*logutilpb.Event{} resp := &vtctldatapb.InitShardPrimaryResponse{} - err = s.InitShardPrimaryLocked(ctx, ev, req, waitReplicasTimeout, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(e *logutilpb.Event) { + err = s.InitShardPrimaryLocked(ctx, ev, req, waitReplicasTimeout, s.tmc, logutil.NewCallbackLogger(func(e *logutilpb.Event) { m.Lock() defer m.Unlock() @@ -1318,6 +1431,9 @@ func (s *VtctldServer) InitShardPrimaryLocked( // PlannedReparentShard is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) PlannedReparentShard(ctx context.Context, req *vtctldatapb.PlannedReparentShardRequest) (*vtctldatapb.PlannedReparentShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.PlannedReparentShard") + defer span.Finish() + waitReplicasTimeout, ok, err := protoutil.DurationFromProto(req.WaitReplicasTimeout) if err != nil { return nil, err @@ -1325,6 +1441,18 @@ func (s *VtctldServer) PlannedReparentShard(ctx context.Context, req *vtctldatap waitReplicasTimeout = time.Second * 30 } + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + + if req.AvoidPrimary != nil { + span.Annotate("avoid_primary_alias", topoproto.TabletAliasString(req.AvoidPrimary)) + } + + if req.NewPrimary != nil { + span.Annotate("new_primary_alias", topoproto.TabletAliasString(req.NewPrimary)) + } + m := sync.RWMutex{} logstream := []*logutilpb.Event{} logger := logutil.NewCallbackLogger(func(e *logutilpb.Event) { @@ -1369,6 +1497,11 @@ func (s *VtctldServer) PlannedReparentShard(ctx context.Context, req *vtctldatap // RebuildVSchemaGraph is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) RebuildVSchemaGraph(ctx context.Context, req *vtctldatapb.RebuildVSchemaGraphRequest) (*vtctldatapb.RebuildVSchemaGraphResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.RebuildVSchemaGraph") + defer span.Finish() + + span.Annotate("cells", strings.Join(req.Cells, ",")) + if err := s.ts.RebuildSrvVSchema(ctx, req.Cells); err != nil { return nil, err } @@ -1378,6 +1511,14 @@ func (s *VtctldServer) RebuildVSchemaGraph(ctx context.Context, req *vtctldatapb // RemoveKeyspaceCell is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) RemoveKeyspaceCell(ctx context.Context, req *vtctldatapb.RemoveKeyspaceCellRequest) (*vtctldatapb.RemoveKeyspaceCellResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.RemoveKeyspaceCell") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("cell", req.Cell) + span.Annotate("force", req.Force) + span.Annotate("recursive", req.Recursive) + shards, err := s.ts.GetShardNames(ctx, req.Keyspace) if err != nil { return nil, err @@ -1402,6 +1543,15 @@ func (s *VtctldServer) RemoveKeyspaceCell(ctx context.Context, req *vtctldatapb. // RemoveShardCell is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) RemoveShardCell(ctx context.Context, req *vtctldatapb.RemoveShardCellRequest) (*vtctldatapb.RemoveShardCellResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.RemoveShardCell") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.ShardName) + span.Annotate("cell", req.Cell) + span.Annotate("force", req.Force) + span.Annotate("recursive", req.Recursive) + if err := removeShardCell(ctx, s.ts, req.Cell, req.Keyspace, req.ShardName, req.Recursive, req.Force); err != nil { return nil, err } @@ -1411,10 +1561,15 @@ func (s *VtctldServer) RemoveShardCell(ctx context.Context, req *vtctldatapb.Rem // ReparentTablet is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) ReparentTablet(ctx context.Context, req *vtctldatapb.ReparentTabletRequest) (*vtctldatapb.ReparentTabletResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ReparentTablet") + defer span.Finish() + if req.Tablet == nil { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "tablet alias must not be nil") } + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.Tablet)) + tablet, err := s.ts.GetTablet(ctx, req.Tablet) if err != nil { return nil, err @@ -1459,6 +1614,12 @@ func (s *VtctldServer) ReparentTablet(ctx context.Context, req *vtctldatapb.Repa // ShardReplicationPositions is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctldatapb.ShardReplicationPositionsRequest) (*vtctldatapb.ShardReplicationPositionsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ShardReplicationPositions") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + tabletInfoMap, err := s.ts.GetTabletMapForShard(ctx, req.Keyspace, req.Shard) if err != nil { return nil, fmt.Errorf("GetTabletMapForShard(%s, %s) failed: %w", req.Keyspace, req.Shard, err) @@ -1490,6 +1651,11 @@ func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctl go func(ctx context.Context, alias string, tablet *topodatapb.Tablet) { defer wg.Done() + span, ctx := trace.NewSpan(ctx, "VtctldServer.getPrimaryPosition") + defer span.Finish() + + span.Annotate("tablet_alias", alias) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -1527,6 +1693,11 @@ func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctl go func(ctx context.Context, alias string, tablet *topodatapb.Tablet) { defer wg.Done() + span, ctx := trace.NewSpan(ctx, "VtctldServer.getReplicationStatus") + defer span.Finish() + + span.Annotate("tablet_alias", alias) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -1570,10 +1741,15 @@ func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctl // TabletExternallyReparented is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) TabletExternallyReparented(ctx context.Context, req *vtctldatapb.TabletExternallyReparentedRequest) (*vtctldatapb.TabletExternallyReparentedResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.TabletExternallyReparented") + defer span.Finish() + if req.Tablet == nil { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "TabletExternallyReparentedRequest.Tablet must not be nil") } + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.Tablet)) + tablet, err := s.ts.GetTablet(ctx, req.Tablet) if err != nil { log.Warningf("TabletExternallyReparented: failed to read tablet record for %v: %v", topoproto.TabletAliasString(req.Tablet), err) @@ -1630,6 +1806,13 @@ func (s *VtctldServer) TabletExternallyReparented(ctx context.Context, req *vtct // UpdateCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) UpdateCellInfo(ctx context.Context, req *vtctldatapb.UpdateCellInfoRequest) (*vtctldatapb.UpdateCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.UpdateCellInfo") + defer span.Finish() + + span.Annotate("cell", req.Name) + span.Annotate("cell_server_address", req.CellInfo.ServerAddress) + span.Annotate("cell_root", req.CellInfo.Root) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -1670,6 +1853,12 @@ func (s *VtctldServer) UpdateCellInfo(ctx context.Context, req *vtctldatapb.Upda // UpdateCellsAlias is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) UpdateCellsAlias(ctx context.Context, req *vtctldatapb.UpdateCellsAliasRequest) (*vtctldatapb.UpdateCellsAliasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.UpdateCellsAlias") + defer span.Finish() + + span.Annotate("cells_alias", req.Name) + span.Annotate("cells_alias_cells", strings.Join(req.CellsAlias.Cells, ",")) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() diff --git a/go/vt/vtctl/grpcvtctldserver/topo.go b/go/vt/vtctl/grpcvtctldserver/topo.go index 4363a0c2174..a8d501d1528 100644 --- a/go/vt/vtctl/grpcvtctldserver/topo.go +++ b/go/vt/vtctl/grpcvtctldserver/topo.go @@ -90,6 +90,14 @@ func deleteShard(ctx context.Context, ts *topo.Server, keyspace string, shard st // distinct from the RemoveShardCell rpc. Despite having similar names, they are // **not** the same! func deleteShardCell(ctx context.Context, ts *topo.Server, keyspace string, shard string, cell string, recursive bool) error { + span, ctx := trace.NewSpan(ctx, "VtctldServer.deleteShardCell") + defer span.Finish() + + span.Annotate("keyspace", keyspace) + span.Annotate("shard", shard) + span.Annotate("cell", cell) + span.Annotate("recursive", recursive) + var aliases []*topodatapb.TabletAlias // Get the ShardReplication object for the cell. Collect all the tablets @@ -165,6 +173,12 @@ func deleteShardCell(ctx context.Context, ts *topo.Server, keyspace string, shar } func deleteTablet(ctx context.Context, ts *topo.Server, alias *topodatapb.TabletAlias, allowPrimary bool) (err error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.deleteTablet") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(alias)) + span.Annotate("allow_primary", allowPrimary) + tablet, err := ts.GetTablet(ctx, alias) if err != nil { return err @@ -175,6 +189,8 @@ func deleteTablet(ctx context.Context, ts *topo.Server, alias *topodatapb.Tablet return err } + span.Annotate("is_primary", isPrimary) + if isPrimary && !allowPrimary { return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot delete tablet %v as it is a master, pass AllowPrimary = true", topoproto.TabletAliasString(alias)) } @@ -218,6 +234,15 @@ func deleteTablet(ctx context.Context, ts *topo.Server, alias *topodatapb.Tablet } func removeShardCell(ctx context.Context, ts *topo.Server, cell string, keyspace string, shardName string, recursive bool, force bool) error { + span, ctx := trace.NewSpan(ctx, "VtctldServer.removeShardCell") + defer span.Finish() + + span.Annotate("keyspace", keyspace) + span.Annotate("shard", shardName) + span.Annotate("cell", cell) + span.Annotate("recursive", recursive) + span.Annotate("force", force) + shard, err := ts.GetShard(ctx, keyspace, shardName) if err != nil { return err From a5633cc980c63f057a25000db3f2ecbee051cad3 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Jun 2021 10:46:31 -0400 Subject: [PATCH 4/4] [vtctldserver] Update `ApplyVSchema` with tracing after rebase Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/server.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 122cc1dceca..8ce8e329d3e 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -25,9 +25,8 @@ import ( "sync" "time" - "google.golang.org/protobuf/proto" - "google.golang.org/grpc" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/event" @@ -152,6 +151,14 @@ func (s *VtctldServer) ApplyRoutingRules(ctx context.Context, req *vtctldatapb.A // ApplyVSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyVSchemaRequest) (*vtctldatapb.ApplyVSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ApplyVSchema") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("cells", strings.Join(req.Cells, ",")) + span.Annotate("skip_rebuild", req.SkipRebuild) + span.Annotate("dry_run", req.DryRun) + if _, err := s.ts.GetKeyspace(ctx, req.Keyspace); err != nil { if topo.IsErrType(err, topo.NoNode) { return nil, vterrors.Wrapf(err, "keyspace(%s) doesn't exist, check if the keyspace is initialized", req.Keyspace) @@ -163,10 +170,14 @@ func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyV return nil, vterrors.New(vtrpc.Code_INVALID_ARGUMENT, "must pass exactly one of req.VSchema and req.Sql") } - var vs *vschemapb.Keyspace - var err error + var ( + vs *vschemapb.Keyspace + err error + ) if req.Sql != "" { + span.Annotate("sql_mode", true) + stmt, err := sqlparser.Parse(req.Sql) if err != nil { return nil, vterrors.Wrapf(err, "Parse(%s)", req.Sql) @@ -186,6 +197,7 @@ func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyV return nil, vterrors.Wrapf(err, "ApplyVSchemaDDL(%s,%v,%v)", req.Keyspace, vs, ddl) } } else { // "jsonMode" + span.Annotate("sql_mode", false) vs = req.VSchema }