From d26d32afe3c0727cb0274d10b5b09d64611c6a79 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 8 Feb 2021 21:06:40 -0500 Subject: [PATCH 1/7] Add `PlannedReparenter` struct to encapsulate the logic involved in a PRS This will eventually replace the equivalent logic in wrangler, which can then delegate into this module, along with `VtctldServer` when we add PRS to that API. Signed-off-by: Andrew Mason --- .../vtctl/reparentutil/planned_reparenter.go | 626 ++++++++++++++++++ 1 file changed, 626 insertions(+) create mode 100644 go/vt/vtctl/reparentutil/planned_reparenter.go diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go new file mode 100644 index 00000000000..54526e44580 --- /dev/null +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -0,0 +1,626 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "context" + "fmt" + "sync" + "time" + + "vitess.io/vitess/go/event" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/concurrency" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vttablet/tmclient" + + logutilpb "vitess.io/vitess/go/vt/proto/logutil" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/proto/vtrpc" +) + +// PlannedReparenter performs PlannedReparentShard operations. +type PlannedReparenter struct { + ts *topo.Server + tmc tmclient.TabletManagerClient + logger logutil.Logger +} + +// PlannedReparentOptions provides optional parameters to PlannedReparentShard +// operations. Options are passed by value, so it is safe for callers to mutate +// resue options structs for multiple calls. +type PlannedReparentOptions struct { + NewPrimaryAlias *topodatapb.TabletAlias + AvoidPrimaryAlias *topodatapb.TabletAlias + WaitReplicasTimeout time.Duration + + // Private options managed internally. We use value-passing semantics to + // set these options inside a PlannedReparent without leaking these details + // back out to the caller. + + lockAction string +} + +// NewPlannedReparenter returns a new PlannedReparenter object, ready to perform +// PlannedReparentShard operations using the given topo.Server, +// TabletManagerClient, and logger. +// +// Providing a nil logger instance is allowed. +func NewPlannedReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, logger logutil.Logger) *PlannedReparenter { + pr := PlannedReparenter{ + ts: ts, + tmc: tmc, + logger: logger, + } + + if pr.logger == nil { + // Create a no-op logger so we can call functions on pr.logger without + // needing to constantly check it for non-nil first. + pr.logger = logutil.NewCallbackLogger(func(e *logutilpb.Event) {}) + } + + return &pr +} + +// ReparentShard performs a PlannedReparentShard operation on the given keyspace +// and shard. It will make the provided tablet the primary for the shard, when +// both the current and desired primary are reachable and in a good state. +func (pr *PlannedReparenter) ReparentShard(ctx context.Context, keyspace string, shard string, opts PlannedReparentOptions) (*events.Reparent, error) { + opts.lockAction = pr.getLockAction(opts) + + ctx, unlock, err := pr.ts.LockShard(ctx, keyspace, shard, opts.lockAction) + if err != nil { + return nil, err + } + + defer unlock(&err) + + if opts.NewPrimaryAlias == nil && opts.AvoidPrimaryAlias == nil { + shardInfo, err := pr.ts.GetShard(ctx, keyspace, shard) + if err != nil { + return nil, err + } + + opts.AvoidPrimaryAlias = shardInfo.MasterAlias + } + + ev := &events.Reparent{} + defer func() { + switch err { + case nil: + event.DispatchUpdate(ev, "finished PlannedReparentShard") + default: + event.DispatchUpdate(ev, "failed PlannedReparentShard: "+err.Error()) + } + }() + + err = pr.reparentShardLocked(ctx, ev, keyspace, shard, opts) + + return ev, err +} + +func (pr *PlannedReparenter) getLockAction(opts PlannedReparentOptions) string { + return fmt.Sprintf( + "PlannedReparentShard(%v, AvoidPrimary = %v)", + topoproto.TabletAliasString(opts.NewPrimaryAlias), + topoproto.TabletAliasString(opts.AvoidPrimaryAlias), + ) +} + +// prelightChecks checks some invariants that pr.reparentShardLocked() depends +// on. It returns a boolean to indicate if the reparent is a no-op (which +// happens iff the caller specified an AvoidPrimaryAlias and it's not the shard +// primary), as well as an error. +// +// It will also set the NewPrimaryAlias option if the caller did not specify +// one, provided it can choose a new primary candidate. See ChooseNewPrimary() +// for details on primary candidate selection. +func (pr *PlannedReparenter) preflightChecks( + ctx context.Context, + ev *events.Reparent, + keyspace string, + shard string, + tabletMap map[string]*topo.TabletInfo, + opts *PlannedReparentOptions, // we take a pointer here to set NewPrimaryAlias +) (isNoop bool, err error) { + if topoproto.TabletAliasEqual(opts.NewPrimaryAlias, opts.AvoidPrimaryAlias) { + return true, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary-elect tablet %v is the same as the tablet to avoid", topoproto.TabletAliasString(opts.NewPrimaryAlias)) + } + + if opts.NewPrimaryAlias == nil { + if !topoproto.TabletAliasEqual(opts.AvoidPrimaryAlias, ev.ShardInfo.MasterAlias) { + event.DispatchUpdate(ev, "current primary is different than AvoidPrimary, nothing to do") + return true, nil + } + + event.DispatchUpdate(ev, "searching for primary candidate") + + opts.NewPrimaryAlias, err = ChooseNewPrimary(ctx, pr.tmc, &ev.ShardInfo, tabletMap, opts.AvoidPrimaryAlias, opts.WaitReplicasTimeout, pr.logger) + if err != nil { + return true, err + } + + if opts.NewPrimaryAlias == nil { + return true, vterrors.Errorf(vtrpc.Code_INTERNAL, "cannot find a tablet to reparent to") + } + + pr.logger.Infof("elected new primary candidate %v", topoproto.TabletAliasString(opts.NewPrimaryAlias)) + event.DispatchUpdate(ev, "elected new primary candidate") + } + + primaryElectAliasStr := topoproto.TabletAliasString(opts.NewPrimaryAlias) + + newPrimaryTabletInfo, ok := tabletMap[primaryElectAliasStr] + if !ok { + return true, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary-elect tablet %v is not in the shard", primaryElectAliasStr) + } + + ev.NewMaster = *newPrimaryTabletInfo.Tablet + + if topoproto.TabletAliasIsZero(ev.ShardInfo.MasterAlias) { + return true, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "the shard has no current primary, use EmergencyReparentShard instead") + } + + return false, nil +} + +func (pr *PlannedReparenter) performGracefulPromotion( + ctx context.Context, + ev *events.Reparent, + keyspace string, + shard string, + currentPrimary *topo.TabletInfo, + primaryElect topodatapb.Tablet, + tabletMap map[string]*topo.TabletInfo, + opts PlannedReparentOptions, +) (string, error) { + primaryElectAliasStr := topoproto.TabletAliasString(primaryElect.Alias) + ev.OldMaster = *currentPrimary.Tablet + + // Before demoting the old primary, we're going to ensure that replication + // is working from the old primary to the primary-elect. If replication is + // not working, a PlannedReparent is not safe to do, because the candidate + // won't catch up and we'll potentially miss transactions. + pr.logger.Infof("checking replication on primary-elect %v", primaryElectAliasStr) + + // First, we find the position of the current primary. Note that this is + // just a snapshot of the position, since we let it keep accepting writes + // until we're sure we want to proceed with the promotion. + snapshotCtx, snapshotCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer snapshotCancel() + + snapshotPos, err := pr.tmc.MasterPosition(snapshotCtx, currentPrimary.Tablet) + if err != nil { + return "", vterrors.Wrapf(err, "cannot get replication position on current primary %v; current primary must be healthy to perform PlannedReparent", currentPrimary.AliasString()) + } + + // Next, we wait for the primary-elect to catch up to that snapshot point. + // If it can catch up within WaitReplicasTimeout, we can be fairly + // confident that it will catch up on everything else that happens between + // the snapshot point we grabbed above and when we demote the old primary + // below. + // + // We do this as an idempotent SetMaster to make sure the replica knows who + // the current primary is. + setMasterCtx, setMasterCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) + defer setMasterCancel() + + if err := pr.tmc.SetMaster(setMasterCtx, &primaryElect, currentPrimary.Alias, 0, snapshotPos, true); err != nil { + return "", vterrors.Wrapf(err, "replication on primary-elect %v did not catch up in time; replication must be healthy to perform PlannedReparent", primaryElectAliasStr) + } + + // Verify we still have the topology lock before doing the demotion. + if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { + return "", vterrors.Wrap(err, "lost topology lock; aborting") + } + + // Next up, demote the current primary and get its replication position. + // It's fine if the current primary was already demoted, since DemoteMaster + // is idempotent. + pr.logger.Infof("demoting current primary: %v", currentPrimary.AliasString()) + event.DispatchUpdate(ev, "demoting old primary") + + demoteCtx, demoteCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer demoteCancel() + + masterStatus, err := pr.tmc.DemoteMaster(demoteCtx, currentPrimary.Tablet) + if err != nil { + return "", vterrors.Wrapf(err, "failed to DemoteMaster on current primary %v: %v", currentPrimary.AliasString(), err) + } + + // Wait for the primary-elect to catch up to the position we demoted the + // current primary at. If it fails to catch up within WaitReplicasTimeout, + // we will try to roll back to the original primary before aborting. + waitCtx, waitCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) + defer waitCancel() + + waitErr := pr.tmc.WaitForPosition(waitCtx, &primaryElect, masterStatus.Position) + + // Do some wrapping of errors to get the right codes and callstacks. + var finalWaitErr error + switch { + case waitErr != nil: + finalWaitErr = vterrors.Wrapf(waitErr, "primary-elect tablet %v failed to catch up with replication %v", primaryElectAliasStr, masterStatus.Position) + case ctx.Err() == context.DeadlineExceeded: + finalWaitErr = vterrors.New(vtrpc.Code_DEADLINE_EXCEEDED, "PlannedReparent timed out; please try again") + } + + if finalWaitErr != nil { + // It's possible that we've used up the calling context's timeout, or + // that not enough time is left on the it to finish the rollback. + // We create a new background context to avoid a partial rollback, which + // could leave the cluster in a worse state than when we started. + undoCtx, undoCancel := context.WithTimeout(context.Background(), *topo.RemoteOperationTimeout) + defer undoCancel() + + if undoErr := pr.tmc.UndoDemoteMaster(undoCtx, currentPrimary.Tablet); undoErr != nil { + pr.logger.Warningf("encountered error while performing UndoDemoteMaster(%v): %v", currentPrimary.AliasString(), undoErr) + finalWaitErr = vterrors.Wrapf(finalWaitErr, "encountered error while performing UndoDemoteMaster(%v): %v", currentPrimary.AliasString(), undoErr) + } + + return "", finalWaitErr + } + + // Primary-elect is caught up to the current primary. We can do the + // promotion now. + promoteCtx, promoteCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) + defer promoteCancel() + + rp, err := pr.tmc.PromoteReplica(promoteCtx, &primaryElect) + if err != nil { + return "", vterrors.Wrapf(err, "primary-elect tablet %v failed to be promoted to primary; please try again", primaryElectAliasStr) + } + + if ctx.Err() == context.DeadlineExceeded { + // PromoteReplica succeeded, but we ran out of time. PRS needs to be + // re-run to complete fully. + return "", vterrors.Errorf(vtrpc.Code_DEADLINE_EXCEEDED, "PLannedReparent timed out after successfully promoting primary-elect %v; please re-run to fix up the replicas", primaryElectAliasStr) + } + + return rp, nil +} + +func (pr *PlannedReparenter) performPartialPromotionRecovery(ctx context.Context, primaryElect topodatapb.Tablet) (string, error) { + // It's possible that a previous attempt to reparent failed to SetReadWrite, + // so call it here to make sure the underlying MySQL is read-write on the + // candidate primary. + setReadWriteCtx, setReadWriteCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer setReadWriteCancel() + + if err := pr.tmc.SetReadWrite(setReadWriteCtx, &primaryElect); err != nil { + return "", vterrors.Wrapf(err, "failed to SetReadWrite on current primary %v", topoproto.TabletAliasString(primaryElect.Alias)) + } + + // The primary is already the one we want according to its tablet record. + refreshCtx, refreshCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer refreshCancel() + + // Get the replication position so we can try to fix the replicas (back in + // reparentShardLocked()) + reparentJournalPosition, err := pr.tmc.MasterPosition(refreshCtx, &primaryElect) + if err != nil { + return "", vterrors.Wrapf(err, "failed to get replication position of current primary %v", topoproto.TabletAliasString(primaryElect.Alias)) + } + + return reparentJournalPosition, nil +} + +func (pr *PlannedReparenter) performPotentialPromotion( + ctx context.Context, + keyspace string, + shard string, + primaryElect topodatapb.Tablet, + tabletMap map[string]*topo.TabletInfo, + opts PlannedReparentOptions, +) (string, error) { + primaryElectAliasStr := topoproto.TabletAliasString(primaryElect.Alias) + + pr.logger.Infof("no clear winner found for current master term; checking if it's safe to recover by electing %v", primaryElectAliasStr) + + type tabletPos struct { + alias string + tablet *topodatapb.Tablet + pos mysql.Position + } + + positions := make(chan tabletPos, len(tabletMap)) + + // First, stop the world, to ensure no writes are happening anywhere. We + // don't trust that we know which tablets might be acting as primaries, so + // we simply demote everyone. + // + // Unlike the normal, single-primary case, we don't try to undo this if we + // fail. If we've made it here, it means there is no clear primary, so we + // don't know who it's safe to roll back to. Leaving everything read-only is + // probably safer, or at least no worse, than whatever weird state we were + // in before. + // + // If any tablets are unreachable, we can't be sure it's safe either, + // because one of the unreachable tablets might have a replication position + // further ahead than the candidate primary. + + var ( + stopAllWg sync.WaitGroup + rec concurrency.AllErrorRecorder + ) + + stopAllCtx, stopAllCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer stopAllCancel() + + for alias, tabletInfo := range tabletMap { + stopAllWg.Add(1) + + go func(alias string, tablet *topodatapb.Tablet) { + defer stopAllWg.Done() + + // Regardless of what type this tablet thinks it is, we will always + // call DemoteMaster to ensure the underlying MySQL server is in + // read-only, and to check its replication position. DemoteMaster is + // idempotent, so it's fine to call it on a replica (or other + // tablet type), that's already in read-only. + pr.logger.Infof("demoting tablet %v", alias) + + masterStatus, err := pr.tmc.DemoteMaster(stopAllCtx, tablet) + if err != nil { + rec.RecordError(vterrors.Wrapf(err, "DemoteMaster(%v) failed on contested primary", alias)) + + return + } + + pos, err := mysql.DecodePosition(masterStatus.Position) + if err != nil { + rec.RecordError(vterrors.Wrapf(err, "cannot decode replication position (%v) for demoted tablet %v", masterStatus.Position, alias)) + + return + } + + positions <- tabletPos{ + alias: alias, + tablet: tablet, + pos: pos, + } + }(alias, tabletInfo.Tablet) + } + + stopAllWg.Wait() + close(positions) + + if rec.HasErrors() { + return "", vterrors.Wrap(rec.Error(), "failed to demote all tablets") + } + + // Construct a mapping of alias to tablet position. + tabletPosMap := make(map[string]tabletPos, len(tabletMap)) + for tp := range positions { + tabletPosMap[tp.alias] = tp + } + + // Make sure no tablet has a more advanced position than the candidate + // primary. It's up to the caller to choose a suitable candidate, and to + // choose another if this check fails. + // + // Note that we still allow replication to run during this time, but we + // assume that no new high water mark can appear because we just demoted all + // tablets to read-only, so there should be no new transactions. + // + // TODO: consider temporarily replicating from another tablet to catch up, + // if the candidate primary is behind that tablet. + tp, ok := tabletPosMap[primaryElectAliasStr] + if !ok { + return "", vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary-elect tablet %v not found in tablet map", primaryElectAliasStr) + } + + primaryElectPos := tp.pos + + for _, tp := range tabletPosMap { + // The primary-elect pos has to be at least as advanced as every tablet + // in the shard. + if !primaryElectPos.AtLeast(tp.pos) { + return "", vterrors.Errorf( + vtrpc.Code_FAILED_PRECONDITION, + "tablet %v (position: %v) contains transactions not found in primary-elect %v (position: %v)", + tp.alias, tp.pos, primaryElectAliasStr, primaryElectPos, + ) + } + } + + // Check that we still have the topology lock. + if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { + return "", vterrors.Wrap(err, "lost topology lock; aborting") + } + + // Promote the candidate primary to type:MASTER. + promoteCtx, promoteCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer promoteCancel() + + rp, err := pr.tmc.PromoteReplica(promoteCtx, &primaryElect) + if err != nil { + return "", vterrors.Wrapf(err, "failed to promote %v to primary", primaryElectAliasStr) + } + + return rp, nil +} + +func (pr *PlannedReparenter) reparentShardLocked( + ctx context.Context, + ev *events.Reparent, + keyspace string, + shard string, + opts PlannedReparentOptions, +) error { + shardInfo, err := pr.ts.GetShard(ctx, keyspace, shard) + if err != nil { + return err + } + + ev.ShardInfo = *shardInfo + + event.DispatchUpdate(ev, "reading tablet map") + + tabletMap, err := pr.ts.GetTabletMapForShard(ctx, keyspace, shard) + if err != nil { + return err + } + + // Check invariants that PlannedReparentShard depends on. + if isNoop, err := pr.preflightChecks(ctx, ev, keyspace, shard, tabletMap, &opts); err != nil { + return err + } else if isNoop { + return nil + } + + currentPrimary := FindCurrentPrimary(tabletMap, pr.logger) + reparentJournalPos := "" + + // Depending on whether we can find a current primary, and what the caller + // specified as the candidate primary, we will do one of three kinds of + // promotions: + // + // 1) There is no clear current primary. In this case we will try to + // determine if it's safe to promote the candidate specified by the caller. + // If it's not -- including if any tablet in the shard is unreachable -- we + // bail. We also don't attempt to rollback a failed demotion in this case. + // + // 2) The current primary is the same as the candidate primary specified by + // the caller. In this case, we assume there was a previous PRS for this + // primary, and the caller is re-issuing the call to fix-up any replicas. We + // also idempotently set the desired primary as read-write, just in case. + // + // 3) The current primary and the desired primary differ. In this case, we + // perform a graceful promotion, in which we validate the desired primary is + // sufficiently up-to-date, demote the current primary, wait for the desired + // primary to catch up to that position, and set the desired primary + // read-write. We will attempt to rollback a failed demotion in this case, + // unlike in case (1), because we have a known good state to rollback to. + // + // In all cases, we will retrieve the reparent journal position that was + // inserted in the new primary's journal, so we can use it below to check + // that all the replicas have attached to new primary successfully. + switch { + case currentPrimary == nil: + // Case (1): no clear current primary. Try to find a safe promotion + // candidate, and promote to it. + reparentJournalPos, err = pr.performPotentialPromotion(ctx, keyspace, shard, ev.NewMaster, tabletMap, opts) + case topoproto.TabletAliasEqual(currentPrimary.Alias, opts.NewPrimaryAlias): + // Case (2): desired new primary is the current primary. Attempt to fix + // up replicas to recover from a previous partial promotion. + reparentJournalPos, err = pr.performPartialPromotionRecovery(ctx, ev.NewMaster) + default: + // Case (3): desired primary and current primary differ. Do a graceful + // demotion-then-promotion. + reparentJournalPos, err = pr.performGracefulPromotion(ctx, ev, keyspace, shard, currentPrimary, ev.NewMaster, tabletMap, opts) + } + + if err != nil { + return err + } + + if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { + return vterrors.Wrap(err, "lost topology lock, aborting") + } + + if err := pr.reparentTablets(ctx, ev, reparentJournalPos, tabletMap, opts); err != nil { + return err + } + + return nil +} + +func (pr *PlannedReparenter) reparentTablets( + ctx context.Context, + ev *events.Reparent, + reparentJournalPosition string, + tabletMap map[string]*topo.TabletInfo, + opts PlannedReparentOptions, +) error { + // Create a cancellable context for the entire set of reparent operations. + // If any error conditions happen, we can cancel all outgoing RPCs. + replCtx, replCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) + defer replCancel() + + // Go thorugh all the tablets. + // - New primary: populate the reparent journal. + // - Everybody else: reparent to the new primary; wait for the reparent + // journal row. + event.DispatchUpdate(ev, "reparenting all tablets") + + // We add a (hopefully) unique record to the reparent journal table on the + // new primary, so we can check if replicas got it through replication. + reparentJournalTimestamp := time.Now().UnixNano() + primaryElectAliasStr := topoproto.TabletAliasString(ev.NewMaster.Alias) + replicasWg := sync.WaitGroup{} + rec := concurrency.AllErrorRecorder{} + + // Point all replicas at the new primary and check that they receive the + // reparent journal entry, proving that they are replicating from the new + // primary. We do this concurrently with adding the journal entry (after + // this loop), because if semi-sync is enabled, the update to the journal + // table will block until at least one replica is successfully attached to + // the new primary. + for alias, tabletInfo := range tabletMap { + if alias == primaryElectAliasStr { + continue + } + + go func(alias string, tablet *topodatapb.Tablet) { + defer replicasWg.Done() + pr.logger.Infof("setting new primary on replica %v", alias) + + // Note: we used to force replication to start on the old primary, + // but now that we support "resuming" a previously-failed PRS + // attempt, we can no longer assume that we know who the former + // primary was. Instead, we rely on the former primary to remember + // that it needs to start replication after transitioning from + // MASTER => REPLICA. + forceStartReplication := false + if err := pr.tmc.SetMaster(replCtx, tablet, ev.NewMaster.Alias, reparentJournalTimestamp, reparentJournalPosition, forceStartReplication); err != nil { + rec.RecordError(vterrors.Wrapf(err, "tablet %v failed SetMaster(%v): %v", alias, primaryElectAliasStr, err)) + } + }(alias, tabletInfo.Tablet) + } + + // Add a reparent journal entry on the new primary. If semi-sync is enabled, + // this blocks until at least one replica is reparented (above) and + // successfully replicating from the new primary. + // + // If we fail to populate the reparent journal, there's no way the replicas + // will work, so we cancel the ongoing reparent RPCs and bail out. + pr.logger.Infof("populating reparent journal on new primary %v", primaryElectAliasStr) + if err := pr.tmc.PopulateReparentJournal(replCtx, &ev.NewMaster, reparentJournalTimestamp, "PlannedReparentShard", ev.NewMaster.Alias, reparentJournalPosition); err != nil { + pr.logger.Warningf("primary failed to PopulateReparentJournal (position: %v); cancelling replica reparent attempts", reparentJournalPosition) + replCancel() + replicasWg.Wait() + + return vterrors.Wrapf(err, "failed PopulateReparentJournal(primary=%v, ts=%v, pos=%v): %v", primaryElectAliasStr, reparentJournalTimestamp, reparentJournalPosition, err) + } + + // Reparent journal has been populated on the new primary. We just need to + // wait for all the replicas to receive it. + replicasWg.Wait() + + if err := rec.Error(); err != nil { + msg := "some replicas failed to reparent; retry PlannedReparentShard with the same new primary alias (%v) to retry failed replicas" + pr.logger.Errorf2(err, msg, primaryElectAliasStr) + return vterrors.Wrapf(err, msg, primaryElectAliasStr) + } + + return nil +} From 3a6fc8b7993c218d9308580fe2304d1d7efdad11 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 14 Feb 2021 11:39:58 -0500 Subject: [PATCH 2/7] Add `ForceSetShardMaster` option to `AddTablet*` testutil functions This allows conveniently setting up a shard with multiple tablets claiming to be MASTER, with the shard be correctly configured to have _a_ serving master. Previously, those were mutually-exclusive setups with `AddTablets`. Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/testutil/util.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/util.go b/go/vt/vtctl/grpcvtctldserver/testutil/util.go index 290e153fe45..ba1b2d09f67 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/util.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/util.go @@ -20,6 +20,7 @@ package testutil import ( "context" + "errors" "fmt" "testing" @@ -87,6 +88,10 @@ type AddTabletOptions struct { // update the shard record to make that tablet the primary, and fail the // test if the shard record has a serving primary already. AlsoSetShardMaster bool + // ForceSetShardMaster, when combined with AlsoSetShardMaster, will ignore + // any existing primary in the shard, making the current tablet the serving + // primary (given it is type MASTER), and log that it has done so. + ForceSetShardMaster bool // SkipShardCreation, when set, makes AddTablet never attempt to create a // shard record in the topo under any circumstances. SkipShardCreation bool @@ -136,7 +141,13 @@ func AddTablet(ctx context.Context, t *testing.T, ts *topo.Server, tablet *topod if tablet.Type == topodatapb.TabletType_MASTER && opts.AlsoSetShardMaster { _, err := ts.UpdateShardFields(ctx, tablet.Keyspace, tablet.Shard, func(si *topo.ShardInfo) error { if si.IsMasterServing && si.MasterAlias != nil { - return fmt.Errorf("shard %v/%v already has a serving master (%v)", tablet.Keyspace, tablet.Shard, topoproto.TabletAliasString(si.MasterAlias)) + msg := fmt.Sprintf("shard %v/%v already has a serving master (%v)", tablet.Keyspace, tablet.Shard, topoproto.TabletAliasString(si.MasterAlias)) + + if !opts.ForceSetShardMaster { + return errors.New(msg) + } + + t.Logf("%s; replacing with %v because ForceSetShardMaster = true", msg, topoproto.TabletAliasString(tablet.Alias)) } si.MasterAlias = tablet.Alias From e7562fe6cdbb08089d8c23b56fd97aee3fce60ca Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 9 Feb 2021 20:47:11 -0500 Subject: [PATCH 3/7] Add PlannedReparenter tests - add preflightChecks tests - add test cases for `performPartialPromotionRecovery` - remove unused arg, add test cases for `performPotentialPromotion` - add missing call to waitgroup add - add test cases for `reparentTablets` - add tests for graceful promotion - add the rest of the PlannedReparenter tests Signed-off-by: Andrew Mason --- .../reparentutil/emergency_reparenter_test.go | 209 ++ .../vtctl/reparentutil/planned_reparenter.go | 5 +- .../reparentutil/planned_reparenter_test.go | 3184 +++++++++++++++++ 3 files changed, 3396 insertions(+), 2 deletions(-) create mode 100644 go/vt/vtctl/reparentutil/planned_reparenter_test.go diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index d948c47b2c9..95ba0b9e885 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1650,18 +1650,47 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { type emergencyReparenterTestTMClient struct { tmclient.TabletManagerClient + // keyed by tablet alias. + DemoteMasterDelays map[string]time.Duration + // keyed by tablet alias. + DemoteMasterResults map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + } + // keyed by tablet alias. + MasterPositionDelays map[string]time.Duration + // keyed by tablet alias. + MasterPositionResults map[string]struct { + Position string + Error error + } + // keyed by tablet alias. + PopulateReparentJournalDelays map[string]time.Duration // keyed by tablet alias PopulateReparentJournalResults map[string]error // keyed by tablet alias. + PromoteReplicaDelays map[string]time.Duration + // keyed by tablet alias. injects a sleep to the end of the function + // regardless of parent context timeout or error result. + PromoteReplicaPostDelays map[string]time.Duration + // keyed by tablet alias. PromoteReplicaResults map[string]struct { Result string Error error } + ReplicationStatusResults map[string]struct { + Position *replicationdatapb.Status + Error error + } // keyed by tablet alias. SetMasterDelays map[string]time.Duration // keyed by tablet alias. SetMasterResults map[string]error // keyed by tablet alias. + SetReadWriteDelays map[string]time.Duration + // keyed by tablet alias. + SetReadWriteResults map[string]error + // keyed by tablet alias. StopReplicationAndGetStatusDelays map[string]time.Duration // keyed by tablet alias. StopReplicationAndGetStatusResults map[string]struct { @@ -1671,9 +1700,74 @@ type emergencyReparenterTestTMClient struct { } // keyed by tablet alias. WaitForPositionDelays map[string]time.Duration + // keyed by tablet alias. injects a sleep to the end of the function + // regardless of parent context timeout or error result. + WaitForPositionPostDelays map[string]time.Duration // WaitForPosition(tablet *topodatapb.Tablet, position string) error, so we // key by tablet alias and then by position. WaitForPositionResults map[string]map[string]error + // keyed by tablet alias. + UndoDemoteMasterDelays map[string]time.Duration + // keyed by tablet alias + UndoDemoteMasterResults map[string]error +} + +func (fake *emergencyReparenterTestTMClient) DemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.MasterStatus, error) { + if fake.DemoteMasterResults == nil { + return nil, assert.AnError + } + + if tablet.Alias == nil { + return nil, assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.DemoteMasterDelays != nil { + if delay, ok := fake.DemoteMasterDelays[key]; ok { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.DemoteMasterResults[key]; ok { + return result.Status, result.Error + } + + return nil, assert.AnError +} + +func (fake *emergencyReparenterTestTMClient) MasterPosition(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + if fake.MasterPositionResults == nil { + return "", assert.AnError + } + + if tablet.Alias == nil { + return "", assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.MasterPositionDelays != nil { + if delay, ok := fake.MasterPositionDelays[key]; ok { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.MasterPositionResults[key]; ok { + return result.Position, result.Error + } + + return "", assert.AnError } func (fake *emergencyReparenterTestTMClient) PopulateReparentJournal(ctx context.Context, tablet *topodatapb.Tablet, timeCreatedNS int64, actionName string, primaryAlias *topodatapb.TabletAlias, pos string) error { @@ -1682,6 +1776,17 @@ func (fake *emergencyReparenterTestTMClient) PopulateReparentJournal(ctx context } key := topoproto.TabletAliasString(tablet.Alias) + + if fake.PopulateReparentJournalDelays != nil { + if delay, ok := fake.PopulateReparentJournalDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } if result, ok := fake.PopulateReparentJournalResults[key]; ok { return result } @@ -1695,6 +1800,28 @@ func (fake *emergencyReparenterTestTMClient) PromoteReplica(ctx context.Context, } key := topoproto.TabletAliasString(tablet.Alias) + + defer func() { + if fake.PromoteReplicaPostDelays == nil { + return + } + + if delay, ok := fake.PromoteReplicaPostDelays[key]; ok { + time.Sleep(delay) + } + }() + + if fake.PromoteReplicaDelays != nil { + if delay, ok := fake.PromoteReplicaDelays[key]; ok { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + if result, ok := fake.PromoteReplicaResults[key]; ok { return result.Result, result.Error } @@ -1702,6 +1829,20 @@ func (fake *emergencyReparenterTestTMClient) PromoteReplica(ctx context.Context, return "", assert.AnError } +func (fake *emergencyReparenterTestTMClient) ReplicationStatus(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.Status, error) { + if fake.ReplicationStatusResults == nil { + return nil, assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if result, ok := fake.ReplicationStatusResults[key]; ok { + return result.Position, result.Error + } + + return nil, assert.AnError +} + func (fake *emergencyReparenterTestTMClient) SetMaster(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { if fake.SetMasterResults == nil { return assert.AnError @@ -1727,6 +1868,35 @@ func (fake *emergencyReparenterTestTMClient) SetMaster(ctx context.Context, tabl return assert.AnError } +func (fake *emergencyReparenterTestTMClient) SetReadWrite(ctx context.Context, tablet *topodatapb.Tablet) error { + if fake.SetReadWriteResults == nil { + return assert.AnError + } + + if tablet.Alias == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.SetReadWriteDelays != nil { + if delay, ok := fake.SetReadWriteDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if err, ok := fake.SetReadWriteResults[key]; ok { + return err + } + + return assert.AnError +} + func (fake *emergencyReparenterTestTMClient) StopReplicationAndGetStatus(ctx context.Context, tablet *topodatapb.Tablet, mode replicationdatapb.StopReplicationMode) (*replicationdatapb.Status, *replicationdatapb.StopReplicationStatus, error) { if fake.StopReplicationAndGetStatusResults == nil { return nil, nil, assert.AnError @@ -1759,6 +1929,16 @@ func (fake *emergencyReparenterTestTMClient) StopReplicationAndGetStatus(ctx con func (fake *emergencyReparenterTestTMClient) WaitForPosition(ctx context.Context, tablet *topodatapb.Tablet, position string) error { tabletKey := topoproto.TabletAliasString(tablet.Alias) + defer func() { + if fake.WaitForPositionPostDelays == nil { + return + } + + if delay, ok := fake.WaitForPositionPostDelays[tabletKey]; ok { + time.Sleep(delay) + } + }() + if fake.WaitForPositionDelays != nil { if delay, ok := fake.WaitForPositionDelays[tabletKey]; ok { select { @@ -1786,3 +1966,32 @@ func (fake *emergencyReparenterTestTMClient) WaitForPosition(ctx context.Context return result } + +func (fake *emergencyReparenterTestTMClient) UndoDemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) error { + if fake.UndoDemoteMasterResults == nil { + return assert.AnError + } + + if tablet.Alias == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.UndoDemoteMasterDelays != nil { + if delay, ok := fake.UndoDemoteMasterDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.UndoDemoteMasterResults[key]; ok { + return result + } + + return assert.AnError +} diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go index 54526e44580..3873984d9e7 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -329,7 +329,6 @@ func (pr *PlannedReparenter) performPotentialPromotion( shard string, primaryElect topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, - opts PlannedReparentOptions, ) (string, error) { primaryElectAliasStr := topoproto.TabletAliasString(primaryElect.Alias) @@ -518,7 +517,7 @@ func (pr *PlannedReparenter) reparentShardLocked( case currentPrimary == nil: // Case (1): no clear current primary. Try to find a safe promotion // candidate, and promote to it. - reparentJournalPos, err = pr.performPotentialPromotion(ctx, keyspace, shard, ev.NewMaster, tabletMap, opts) + reparentJournalPos, err = pr.performPotentialPromotion(ctx, keyspace, shard, ev.NewMaster, tabletMap) case topoproto.TabletAliasEqual(currentPrimary.Alias, opts.NewPrimaryAlias): // Case (2): desired new primary is the current primary. Attempt to fix // up replicas to recover from a previous partial promotion. @@ -580,6 +579,8 @@ func (pr *PlannedReparenter) reparentTablets( continue } + replicasWg.Add(1) + go func(alias string, tablet *topodatapb.Tablet) { defer replicasWg.Done() pr.logger.Infof("setting new primary on replica %v", alias) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter_test.go b/go/vt/vtctl/reparentutil/planned_reparenter_test.go new file mode 100644 index 00000000000..fccf7da03cb --- /dev/null +++ b/go/vt/vtctl/reparentutil/planned_reparenter_test.go @@ -0,0 +1,3184 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" + "vitess.io/vitess/go/vt/vttablet/tmclient" + + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + "vitess.io/vitess/go/vt/proto/vttime" +) + +func TestNewPlannedReparenter(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + logger logutil.Logger + }{ + { + name: "default case", + logger: logutil.NewMemoryLogger(), + }, + { + name: "overrides nil logger with no-op", + logger: nil, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + er := NewPlannedReparenter(nil, nil, tt.logger) + assert.NotNil(t, er.logger, "NewPlannedReparenter should never result in a nil logger instance on the EmergencyReparenter") + }) + } +} + +func TestPlannedReparenter_ReparentShard(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ts *topo.Server + tmc tmclient.TabletManagerClient + tablets []*topodatapb.Tablet + lockShardBeforeTest bool + + keyspace string + shard string + opts PlannedReparentOptions + + expectedEvent *events.Reparent + shouldErr bool + }{ + { + name: "success", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "position1", + Error: nil, + }, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + + shouldErr: false, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + }, + { + name: "cannot lock shard", + ts: memorytopo.NewServer("zone1"), + tmc: nil, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + lockShardBeforeTest: true, + + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{}, + + expectedEvent: nil, + shouldErr: true, + }, + { + // The simplest setup required to make an overall ReparentShard call + // fail is to set NewPrimaryAlias = AvoidPrimaryAlias, which will + // fail the preflight checks. Other functions are unit-tested + // thoroughly to cover all the cases. + name: "reparent fails", + ts: memorytopo.NewServer("zone1"), + tmc: nil, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Keyspace: "testkeyspace", + Shard: "-", + Type: topodatapb.TabletType_MASTER, + }, + }, + + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + IsMasterServing: true, + KeyRange: &topodatapb.KeyRange{}, + }, nil), + }, + shouldErr: true, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := ctx + + testutil.AddTablets(ctx, t, tt.ts, &testutil.AddTabletOptions{ + AlsoSetShardMaster: true, + SkipShardCreation: false, + }, tt.tablets...) + + if tt.lockShardBeforeTest { + lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "locking for test") + require.NoError(t, err, "could not lock %s/%s for test case", tt.keyspace, tt.shard) + + defer func() { + unlock(&err) + require.NoError(t, err, "could not unlock %s/%s after test case", tt.keyspace, tt.shard) + }() + + ctx = lctx + } + + pr := NewPlannedReparenter(tt.ts, tt.tmc, logger) + ev, err := pr.ReparentShard(ctx, tt.keyspace, tt.shard, tt.opts) + if tt.shouldErr { + assert.Error(t, err) + AssertReparentEventsEqual(t, tt.expectedEvent, ev) + + if ev != nil { + assert.Contains(t, ev.Status, "failed PlannedReparentShard", "expected event status to indicate failed PRS") + } + + return + } + + assert.NoError(t, err) + AssertReparentEventsEqual(t, tt.expectedEvent, ev) + assert.Contains(t, ev.Status, "finished PlannedReparentShard", "expected event status to indicate successful PRS") + }) + } +} + +func TestPlannedReparenter_getLockAction(t *testing.T) { + t.Parallel() + + pr := &PlannedReparenter{} + tests := []struct { + name string + opts PlannedReparentOptions + expected string + }{ + { + name: "no options", + opts: PlannedReparentOptions{}, + expected: "PlannedReparentShard(, AvoidPrimary = )", + }, + { + name: "desired primary only", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expected: "PlannedReparentShard(zone1-0000000100, AvoidPrimary = )", + }, + { + name: "avoid-primary only", + opts: PlannedReparentOptions{ + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, + expected: "PlannedReparentShard(, AvoidPrimary = zone1-0000000500)", + }, + { + name: "all options specified", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, + expected: "PlannedReparentShard(zone1-0000000100, AvoidPrimary = zone1-0000000500)", + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + actual := pr.getLockAction(tt.opts) + assert.Equal(t, tt.expected, actual) + }) + } +} + +func TestPlannedReparenter_preflightChecks(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + + ts *topo.Server + tmc tmclient.TabletManagerClient + tablets []*topodatapb.Tablet + + ev *events.Reparent + keyspace string + shard string + tabletMap map[string]*topo.TabletInfo + opts *PlannedReparentOptions + + expectedIsNoop bool + expectedEvent *events.Reparent + expectedOpts *PlannedReparentOptions + shouldErr bool + }{ + { + name: "invariants hold", + ev: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, nil), + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + opts: &PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expectedIsNoop: false, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + shouldErr: false, + }, + { + name: "invariants hold with primary selection", + // (TODO:@ajm188) - Rename this type and unify it with the 4 other + // mock implementations I've written by this point. + tmc: &emergencyReparenterTestTMClient{ + ReplicationStatusResults: map[string]struct { + Position *replicationdatapb.Status + Error error + }{ + "zone1-0000000100": { // most advanced position + Position: &replicationdatapb.Status{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + "zone1-0000000101": { + Position: &replicationdatapb.Status{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, + }, + }, + }, + ev: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, nil), + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000500": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + }, + opts: &PlannedReparentOptions{ + // Avoid the current primary. + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, + expectedIsNoop: false, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + expectedOpts: &PlannedReparentOptions{ + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 500, + }, + // NewPrimaryAlias gets populated by the preflightCheck code + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + shouldErr: false, + }, + { + name: "new-primary and avoid-primary match", + opts: &PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expectedIsNoop: true, + shouldErr: true, + }, + { + name: "current shard primary is not avoid-primary", + ev: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, nil), + }, + opts: &PlannedReparentOptions{ + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + expectedIsNoop: true, // nothing to do, but not an error! + shouldErr: false, + }, + { + // this doesn't cause an actual error from ChooseNewPrimary, because + // the only way to do that is to set AvoidPrimaryAlias == nil, and + // that gets checked in preflightChecks before calling + // ChooseNewPrimary for other reasons. however we do check that we + // get a non-nil result from ChooseNewPrimary in preflightChecks and + // bail out if we don't, so we're forcing that case here. + name: "cannot choose new primary-elect", + ev: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, nil), + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + opts: &PlannedReparentOptions{ + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expectedIsNoop: true, + shouldErr: true, + }, + { + name: "primary-elect is not in tablet map", + ev: &events.Reparent{}, + tabletMap: map[string]*topo.TabletInfo{}, + opts: &PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expectedIsNoop: true, + shouldErr: true, + }, + { + name: "shard has no current primary", + ev: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: nil, + }, nil), + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + opts: &PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expectedIsNoop: true, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: nil, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + shouldErr: true, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + defer func() { + if tt.expectedEvent != nil { + AssertReparentEventsEqualWithMessage(t, tt.expectedEvent, tt.ev, "expected preflightChecks to mutate the passed-in event") + } + + if tt.expectedOpts != nil { + assert.Equal(t, tt.expectedOpts, tt.opts, "expected preflightChecks to mutate the passed in PlannedReparentOptions") + } + }() + + pr := NewPlannedReparenter(tt.ts, tt.tmc, logger) + isNoop, err := pr.preflightChecks(ctx, tt.ev, tt.keyspace, tt.shard, tt.tabletMap, tt.opts) + if tt.shouldErr { + assert.Error(t, err) + assert.Equal(t, tt.expectedIsNoop, isNoop, "preflightChecks returned wrong isNoop signal") + + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedIsNoop, isNoop, "preflightChecks returned wrong isNoop signal") + }) + } +} + +func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ts *topo.Server + tmc tmclient.TabletManagerClient + unlockTopo bool + ctxTimeout time.Duration + + ev *events.Reparent + keyspace string + shard string + currentPrimary *topo.TabletInfo + primaryElect topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + opts PlannedReparentOptions + + expectedPos string + expectedEvent *events.Reparent + shouldErr bool + // Optional function to run some additional post-test assertions. Will + // be run in the main test body before the common assertions are run, + // regardless of the value of tt.shouldErr for that test case. + extraAssertions func(t *testing.T, pos string, err error) + }{ + { + name: "successful promotion", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000200": { + Result: "successful reparent journal position", + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": nil, + }, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + expectedPos: "successful reparent journal position", + shouldErr: false, + }, + { + name: "cannot get snapshot of current primary", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: assert.AnError, + }, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "primary-elect fails to catch up to current primary snapshot position", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": assert.AnError, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "primary-elect times out catching up to current primary snapshot position", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterDelays: map[string]time.Duration{ + "zone1-0000000200": time.Millisecond * 100, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{ + WaitReplicasTimeout: time.Millisecond * 10, + }, + shouldErr: true, + }, + { + name: "lost topology lock", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + }, + unlockTopo: true, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "failed to demote current primary", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Error: assert.AnError, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "primary-elect fails to catch up to current primary demotion position", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": assert.AnError, + }, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "primary-elect times out catching up to current primary demotion position", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionDelays: map[string]time.Duration{ + "zone1-0000000200": time.Millisecond * 100, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": nil, + }, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{ + WaitReplicasTimeout: time.Millisecond * 10, + }, + shouldErr: true, + }, + { + name: "demotion succeeds but parent context times out", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + // This being present means that if we don't encounter a + // a case where either WaitForPosition errors, or the parent + // context times out, then we will fail the test, since it + // will cause the overall function under test to return no + // error. + "zone1-0000000200": { + Result: "success!", + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionPostDelays: map[string]time.Duration{ + "zone1-0000000200": time.Millisecond * 5, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": nil, + }, + }, + }, + ctxTimeout: time.Millisecond * 4, // WaitForPosition won't return error, but will timeout the parent context + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "rollback fails", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": assert.AnError, + }, + }, + UndoDemoteMasterResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + extraAssertions: func(t *testing.T, pos string, err error) { + assert.Contains(t, err.Error(), "UndoDemoteMaster", "expected error to include information about failed demotion rollback") + }, + }, + { + name: "rollback succeeds", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": assert.AnError, + }, + }, + UndoDemoteMasterResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + extraAssertions: func(t *testing.T, pos string, err error) { + assert.NotContains(t, err.Error(), "UndoDemoteMaster", "expected error to not include information about failed demotion rollback") + }, + }, + { + name: "primary-elect fails to promote", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000200": { + Error: assert.AnError, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": nil, + }, + }, + }, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + { + name: "promotion succeeds but parent context times out", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // value of Position doesn't strictly matter for + // this test case, as long as it matches the inner + // key of the WaitForPositionResults map for the + // primary-elect. + Position: "position1", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + }, + PromoteReplicaPostDelays: map[string]time.Duration{ + "zone1-0000000200": time.Millisecond * 20, // 2x the parent context timeout + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000200": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "position1": nil, + }, + }, + }, + ctxTimeout: time.Millisecond * 10, + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + currentPrimary: &topo.TabletInfo{ + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + opts: PlannedReparentOptions{}, + shouldErr: true, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := ctx + + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, + }) + + if !tt.unlockTopo { + lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, err, "could not lock %s/%s for testing", tt.keyspace, tt.shard) + + defer func() { + unlock(&err) + require.NoError(t, err, "could not unlock %s/%s during testing", tt.keyspace, tt.shard) + }() + + ctx = lctx + } + + pr := NewPlannedReparenter(tt.ts, tt.tmc, logger) + + if tt.ctxTimeout > 0 { + _ctx, cancel := context.WithTimeout(ctx, tt.ctxTimeout) + defer cancel() + + ctx = _ctx + } + + pos, err := pr.performGracefulPromotion( + ctx, + tt.ev, + tt.keyspace, + tt.shard, + tt.currentPrimary, + tt.primaryElect, + tt.tabletMap, + tt.opts, + ) + + if tt.extraAssertions != nil { + tt.extraAssertions(t, pos, err) + } + + if tt.shouldErr { + assert.Error(t, err) + + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedPos, pos) + }) + } +} + +func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + tmc tmclient.TabletManagerClient + timeout time.Duration + primaryElect topodatapb.Tablet + expectedPos string + shouldErr bool + }{ + { + name: "successful recovery", + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "position1", + Error: nil, + }, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + expectedPos: "position1", + shouldErr: false, + }, + { + name: "failed to SetReadWrite", + tmc: &emergencyReparenterTestTMClient{ + SetReadWriteResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + shouldErr: true, + }, + { + name: "SetReadWrite timed out", + tmc: &emergencyReparenterTestTMClient{ + SetReadWriteDelays: map[string]time.Duration{ + "zone1-0000000100": time.Millisecond * 50, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + timeout: time.Millisecond * 10, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + shouldErr: true, + }, + { + name: "failed to get MasterPosition from refreshed primary", + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "", + Error: assert.AnError, + }, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + shouldErr: true, + }, + { + name: "MasterPosition timed out", + tmc: &emergencyReparenterTestTMClient{ + MasterPositionDelays: map[string]time.Duration{ + "zone1-0000000100": time.Millisecond * 50, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "position1", + Error: nil, + }, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + timeout: time.Millisecond * 10, + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + shouldErr: true, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := ctx + pr := NewPlannedReparenter(nil, tt.tmc, logger) + + if tt.timeout > 0 { + _ctx, cancel := context.WithTimeout(ctx, tt.timeout) + defer cancel() + + ctx = _ctx + } + + rp, err := pr.performPartialPromotionRecovery(ctx, tt.primaryElect) + if tt.shouldErr { + assert.Error(t, err) + + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedPos, rp, "performPartialPromotionRecovery gave unexpected reparent journal position") + }) + } +} + +func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ts *topo.Server + tmc tmclient.TabletManagerClient + timeout time.Duration + unlockTopo bool + + keyspace string + shard string + primaryElect topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + + expectedPos string + shouldErr bool + }{ + { + name: "success", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000101": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000102": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, + Error: nil, + }, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Result: "reparent journal position", + Error: nil, + }, + }, + }, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + expectedPos: "reparent journal position", + shouldErr: false, + }, + { + name: "failed to DemoteMaster on a tablet", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: nil, + Error: assert.AnError, + }, + }, + }, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + shouldErr: true, + }, + { + name: "timed out during DemoteMaster on a tablet", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterDelays: map[string]time.Duration{ + "zone1-0000000100": time.Millisecond * 50, + }, + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + }, + }, + timeout: time.Millisecond * 10, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + shouldErr: true, + }, + { + name: "failed to DecodePosition on a tablet's demote position", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/this-is-nonsense", + }, + Error: nil, + }, + }, + }, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + shouldErr: true, + }, + { + name: "primary-elect not in tablet map", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{}, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + shouldErr: true, + }, + { + name: "primary-elect not most at most advanced position", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000101": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000102": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10000", + }, + Error: nil, + }, + }, + }, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + shouldErr: true, + }, + { + name: "lost topology lock", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000101": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000102": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + }, + }, + unlockTopo: true, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + shouldErr: true, + }, + { + name: "failed to promote primary-elect", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000101": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000102": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, + Error: nil, + }, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Result: "", + Error: assert.AnError, + }, + }, + }, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + shouldErr: true, + }, + { + name: "timed out while promoting primary-elect", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000101": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000102": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, + Error: nil, + }, + }, + PromoteReplicaDelays: map[string]time.Duration{ + "zone1-0000000100": time.Millisecond * 100, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Result: "reparent journal position", + Error: nil, + }, + }, + }, + timeout: time.Millisecond * 50, + unlockTopo: false, + keyspace: "testkeyspace", + shard: "-", + primaryElect: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + shouldErr: true, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := ctx + pr := NewPlannedReparenter(nil, tt.tmc, logger) + + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, + }) + + if !tt.unlockTopo { + lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, err, "could not lock %s/%s for testing", tt.keyspace, tt.shard) + + defer func() { + unlock(&err) + require.NoError(t, err, "could not unlock %s/%s during testing", tt.keyspace, tt.shard) + }() + + ctx = lctx + } + + if tt.timeout > 0 { + _ctx, cancel := context.WithTimeout(ctx, tt.timeout) + defer cancel() + + ctx = _ctx + } + + rp, err := pr.performPotentialPromotion(ctx, tt.keyspace, tt.shard, tt.primaryElect, tt.tabletMap) + if tt.shouldErr { + assert.Error(t, err) + + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedPos, rp) + }) + } +} + +func TestPlannedReparenter_reparentShardLocked(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ts *topo.Server + tmc tmclient.TabletManagerClient + tablets []*topodatapb.Tablet + unlockTopo bool + + ev *events.Reparent + keyspace string + shard string + opts PlannedReparentOptions + + shouldErr bool + expectedEvent *events.Reparent + }{ + { + name: "success: current primary cannot be determined", // "Case (1)" + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + "zone1-0000000200": { + Status: &replicationdatapb.MasterStatus{ + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000200": nil, // zone1-200 gets promoted + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000200": { + Result: "reparent journal position", + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, // zone1-100 gets reparented under zone1-200 + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + MasterTermStartTime: &vttime.Time{ + Seconds: 1000, + Nanoseconds: 500, + }, + Hostname: "primary1", // claims to be MASTER with same term as primary2 + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_MASTER, + MasterTermStartTime: &vttime.Time{ + Seconds: 1000, + Nanoseconds: 500, + }, + Hostname: "primary2", // claims to be MASTER with same term as primary1 + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ // We want primary2 to be the true primary. + Cell: "zone1", + Uid: 200, + }, + }, + + shouldErr: false, + }, + { + name: "success: current primary is desired primary", // "Case (2)" + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "position1", + Error: nil, + }, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + + shouldErr: false, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + }, + { + name: "success: graceful promotion", // "Case (3)" + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + DemoteMasterResults: map[string]struct { + Status *replicationdatapb.MasterStatus + Error error + }{ + "zone1-0000000100": { + Status: &replicationdatapb.MasterStatus{ + // a few more transactions happen after waiting for replication + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10", + }, + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-8", + Error: nil, + }, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000200": nil, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000200": { + Result: "reparent journal position", + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, // called during reparentTablets to make oldPrimary a replica of newPrimary + "zone1-0000000200": nil, // called during performGracefulPromotion to ensure newPrimary is caught up + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000200": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10": nil, + }, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + MasterTermStartTime: &vttime.Time{ + Seconds: 1000, + Nanoseconds: 500, + }, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + + shouldErr: false, + }, + { + name: "shard not found", + ts: memorytopo.NewServer("zone1"), + tmc: nil, + tablets: nil, + unlockTopo: true, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{}, + + shouldErr: true, + expectedEvent: &events.Reparent{}, + }, + { + name: "preflight checks fail", + ts: memorytopo.NewServer("zone1"), + tmc: nil, + tablets: []*topodatapb.Tablet{ + // Shard has no current primary, so preflight fails. + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + + shouldErr: true, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + }, + { + name: "preflight checks determine PRS is no-op", + ts: memorytopo.NewServer("zone1"), + tmc: nil, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + // This is not the shard primary, so nothing to do. + AvoidPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + }, + + shouldErr: false, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + }, + }, + { + name: "promotion step fails", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + SetReadWriteResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + + shouldErr: true, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + }, + { + name: "lost topology lock", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "position1", + Error: nil, + }, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + unlockTopo: true, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + // This is not the shard primary, so nothing to do. + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + + shouldErr: true, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + }, + { + name: "failed to reparent tablets", + ts: memorytopo.NewServer("zone1"), + tmc: &emergencyReparenterTestTMClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "position1", + Error: nil, + }, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + SetReadWriteResults: map[string]error{ + "zone1-0000000100": nil, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + + ev: &events.Reparent{}, + keyspace: "testkeyspace", + shard: "-", + opts: PlannedReparentOptions{ + NewPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + + shouldErr: true, + expectedEvent: &events.Reparent{ + ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{ + MasterAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + KeyRange: &topodatapb.KeyRange{}, + IsMasterServing: true, + }, nil), + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + Keyspace: "testkeyspace", + Shard: "-", + }, + }, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := ctx + + testutil.AddTablets(ctx, t, tt.ts, &testutil.AddTabletOptions{ + AlsoSetShardMaster: true, + ForceSetShardMaster: true, // Some of our test cases count on having multiple primaries, so let the last one "win". + SkipShardCreation: false, + }, tt.tablets...) + + if !tt.unlockTopo { + lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "locking for testing") + require.NoError(t, err, "could not lock %s/%s for testing", tt.keyspace, tt.shard) + + defer func() { + unlock(&err) + require.NoError(t, err, "error while unlocking %s/%s after test case", tt.keyspace, tt.shard) + }() + + ctx = lctx + } + + if tt.expectedEvent != nil { + defer func() { + AssertReparentEventsEqualWithMessage(t, tt.expectedEvent, tt.ev, "expected reparentShardLocked to mutate the passed-in event") + }() + } + + pr := NewPlannedReparenter(tt.ts, tt.tmc, logger) + + err := pr.reparentShardLocked(ctx, tt.ev, tt.keyspace, tt.shard, tt.opts) + if tt.shouldErr { + assert.Error(t, err) + + return + } + + assert.NoError(t, err) + }) + } +} + +func TestPlannedReparenter_reparentTablets(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + tmc tmclient.TabletManagerClient + + ev *events.Reparent + reparentJournalPosition string + tabletMap map[string]*topo.TabletInfo + opts PlannedReparentOptions + + shouldErr bool + }{ + { + name: "success", + tmc: &emergencyReparenterTestTMClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + "zone1-0000000201": nil, + "zone1-0000000202": nil, + }, + }, + ev: &events.Reparent{ + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + "zone1-0000000200": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000201": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 201, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000202": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 202, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + shouldErr: false, + }, + { + name: "SetMaster failed on replica", + tmc: &emergencyReparenterTestTMClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + "zone1-0000000201": assert.AnError, + "zone1-0000000202": nil, + }, + }, + ev: &events.Reparent{ + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + "zone1-0000000200": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000201": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 201, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000202": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 202, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + shouldErr: true, + }, + { + name: "SetMaster timed out on replica", + tmc: &emergencyReparenterTestTMClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + SetMasterDelays: map[string]time.Duration{ + "zone1-0000000201": time.Millisecond * 50, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + "zone1-0000000201": nil, + "zone1-0000000202": nil, + }, + }, + ev: &events.Reparent{ + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + "zone1-0000000200": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000201": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 201, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000202": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 202, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + opts: PlannedReparentOptions{ + WaitReplicasTimeout: time.Millisecond * 10, + }, + shouldErr: true, + }, + { + name: "PopulateReparentJournal failed out on new primary", + tmc: &emergencyReparenterTestTMClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + "zone1-0000000201": nil, + "zone1-0000000202": nil, + }, + }, + ev: &events.Reparent{ + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + "zone1-0000000200": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000201": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 201, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000202": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 202, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + shouldErr: true, + }, + { + name: "PopulateReparentJournal timed out on new primary", + tmc: &emergencyReparenterTestTMClient{ + PopulateReparentJournalDelays: map[string]time.Duration{ + "zone1-0000000100": time.Millisecond * 50, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + SetMasterResults: map[string]error{ + "zone1-0000000200": nil, + "zone1-0000000201": nil, + "zone1-0000000202": nil, + }, + }, + ev: &events.Reparent{ + NewMaster: topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_MASTER, + }, + }, + "zone1-0000000200": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000201": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 201, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + "zone1-0000000202": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 202, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + opts: PlannedReparentOptions{ + WaitReplicasTimeout: time.Millisecond * 10, + }, + shouldErr: true, + }, + } + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + pr := NewPlannedReparenter(nil, tt.tmc, logger) + err := pr.reparentTablets(ctx, tt.ev, tt.reparentJournalPosition, tt.tabletMap, tt.opts) + if tt.shouldErr { + assert.Error(t, err) + + return + } + + assert.NoError(t, err) + }) + } +} + +// (TODO:@ajm88) when unifying all the mock TMClient implementations (which will +// most likely end up in go/vt/vtctl/testutil), move these to the same testutil +// package. +func AssertReparentEventsEqualWithMessage(t *testing.T, expected *events.Reparent, actual *events.Reparent, msg string) { + t.Helper() + + if msg != "" && !strings.HasSuffix(msg, " ") { + msg = msg + ": " + } + + if expected == nil { + assert.Nil(t, actual, "%sexpected nil Reparent event", msg) + + return + } + + if actual == nil { + // Note: the reason we don't use require.NotNil here is because it would + // fail the entire test, rather than just this one helper, which is + // intended to be an atomic assertion. However, we also don't want to + // have to add a bunch of nil-guards below, as it would complicate the + // code, so we're going to duplicate the nil check to force a failure + // and bail early. + assert.NotNil(t, actual, "%sexpected non-nil Reparent event", msg) + + return + } + + removeVersion := func(si topo.ShardInfo) topo.ShardInfo { + return *topo.NewShardInfo(si.Keyspace(), si.ShardName(), si.Shard, nil) + } + + assert.Equal(t, removeVersion(expected.ShardInfo), removeVersion(actual.ShardInfo), "%sReparent.ShardInfo mismatch", msg) + assert.Equal(t, expected.NewMaster, actual.NewMaster, "%sReparent.NewMaster mismatch", msg) + assert.Equal(t, expected.OldMaster, actual.OldMaster, "%sReparent.OldMaster mismatch", msg) +} + +func AssertReparentEventsEqual(t *testing.T, expected *events.Reparent, actual *events.Reparent) { + t.Helper() + + AssertReparentEventsEqualWithMessage(t, expected, actual, "") +} From ba196348298f6f1b5294d09b90205d67301f38cc Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 15 Feb 2021 21:33:38 -0500 Subject: [PATCH 4/7] Remove legacy PRS implementation in wrangler, delegate to reparentutil Signed-off-by: Andrew Mason --- go/vt/wrangler/reparent.go | 383 ++----------------------------------- 1 file changed, 11 insertions(+), 372 deletions(-) diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 1a7bfe779e3..db3390b08e4 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/event" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" @@ -38,16 +37,14 @@ import ( "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/vterrors" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) const ( - plannedReparentShardOperation = "PlannedReparentShard" + plannedReparentShardOperation = "PlannedReparentShard" //nolint emergencyReparentShardOperation = "EmergencyReparentShard" //nolint tabletExternallyReparentedOperation = "TabletExternallyReparented" //nolint ) @@ -166,378 +163,20 @@ func (wr *Wrangler) InitShardMaster(ctx context.Context, keyspace, shard string, // PlannedReparentShard will make the provided tablet the master for the shard, // when both the current and new master are reachable and in good shape. func (wr *Wrangler) PlannedReparentShard(ctx context.Context, keyspace, shard string, masterElectTabletAlias, avoidMasterAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration) (err error) { - // lock the shard - lockAction := fmt.Sprintf( - "PlannedReparentShard(%v, avoid_master=%v)", - topoproto.TabletAliasString(masterElectTabletAlias), - topoproto.TabletAliasString(avoidMasterAlias)) - ctx, unlock, lockErr := wr.ts.LockShard(ctx, keyspace, shard, lockAction) - if lockErr != nil { - return lockErr - } - defer unlock(&err) - - // Create reusable Reparent event with available info - ev := &events.Reparent{} - - // Attempt to set avoidMasterAlias if not provided by parameters - if masterElectTabletAlias == nil && avoidMasterAlias == nil { - shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard) - if err != nil { - return err - } - avoidMasterAlias = shardInfo.MasterAlias - } + _, err = reparentutil.NewPlannedReparenter(wr.ts, wr.tmc, wr.logger).ReparentShard( + ctx, + keyspace, + shard, + reparentutil.PlannedReparentOptions{ + AvoidPrimaryAlias: avoidMasterAlias, + NewPrimaryAlias: masterElectTabletAlias, + WaitReplicasTimeout: waitReplicasTimeout, + }, + ) - // do the work - err = wr.plannedReparentShardLocked(ctx, ev, keyspace, shard, masterElectTabletAlias, avoidMasterAlias, waitReplicasTimeout) - if err != nil { - event.DispatchUpdate(ev, "failed PlannedReparentShard: "+err.Error()) - } else { - event.DispatchUpdate(ev, "finished PlannedReparentShard") - } return err } -func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, masterElectTabletAlias, avoidMasterTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration) error { - shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard) - if err != nil { - return err - } - ev.ShardInfo = *shardInfo - - event.DispatchUpdate(ev, "reading tablet map") - tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard) - if err != nil { - return err - } - - // Check invariants we're going to depend on. - if topoproto.TabletAliasEqual(masterElectTabletAlias, avoidMasterTabletAlias) { - return fmt.Errorf("master-elect tablet %v is the same as the tablet to avoid", topoproto.TabletAliasString(masterElectTabletAlias)) - } - if masterElectTabletAlias == nil { - if !topoproto.TabletAliasEqual(avoidMasterTabletAlias, shardInfo.MasterAlias) { - event.DispatchUpdate(ev, "current master is different than -avoid_master, nothing to do") - return nil - } - event.DispatchUpdate(ev, "searching for master candidate") - masterElectTabletAlias, err = reparentutil.ChooseNewPrimary(ctx, wr.tmc, shardInfo, tabletMap, avoidMasterTabletAlias, waitReplicasTimeout, wr.logger) - if err != nil { - return err - } - if masterElectTabletAlias == nil { - return fmt.Errorf("cannot find a tablet to reparent to") - } - wr.logger.Infof("elected new master candidate %v", topoproto.TabletAliasString(masterElectTabletAlias)) - event.DispatchUpdate(ev, "elected new master candidate") - } - masterElectTabletAliasStr := topoproto.TabletAliasString(masterElectTabletAlias) - masterElectTabletInfo, ok := tabletMap[masterElectTabletAliasStr] - if !ok { - return fmt.Errorf("master-elect tablet %v is not in the shard", masterElectTabletAliasStr) - } - ev.NewMaster = *masterElectTabletInfo.Tablet - if topoproto.TabletAliasIsZero(shardInfo.MasterAlias) { - return fmt.Errorf("the shard has no master, use EmergencyReparentShard") - } - - // Find the current master (if any) based on the tablet states. We no longer - // trust the shard record for this, because it is updated asynchronously. - currentMaster := reparentutil.FindCurrentPrimary(tabletMap, wr.logger) - - var reparentJournalPos string - - if currentMaster == nil { - // We don't know who the current master is. Either there is no current - // master at all (no tablet claims to be MASTER), or there is no clear - // winner (multiple MASTER tablets with the same timestamp). - // Check if it's safe to promote the selected master candidate. - wr.logger.Infof("No clear winner found for current master term; checking if it's safe to recover by electing %v", masterElectTabletAliasStr) - - // As we contact each tablet, we'll send its replication position here. - type tabletPos struct { - tabletAliasStr string - tablet *topodatapb.Tablet - pos mysql.Position - } - positions := make(chan tabletPos, len(tabletMap)) - - // First stop the world, to ensure no writes are happening anywhere. - // Since we don't trust that we know which tablets might be acting as - // masters, we simply demote everyone. - // - // Unlike the normal, single-master case, we don't try to undo this if - // we bail out. If we're here, it means there is no clear master, so we - // don't know that it's safe to roll back to the previous state. - // Leaving everything read-only is probably safer than whatever weird - // state we were in before. - // - // If any tablets are unreachable, we can't be sure it's safe, because - // one of the unreachable ones might have a replication position farther - // ahead than the candidate master. - wgStopAll := sync.WaitGroup{} - rec := concurrency.AllErrorRecorder{} - - stopAllCtx, stopAllCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer stopAllCancel() - - for tabletAliasStr, tablet := range tabletMap { - wgStopAll.Add(1) - go func(tabletAliasStr string, tablet *topodatapb.Tablet) { - defer wgStopAll.Done() - - // Regardless of what type this tablet thinks it is, we always - // call DemoteMaster to ensure the underlying MySQL is read-only - // and to check its replication position. DemoteMaster is - // idempotent so it's fine to call it on a replica that's - // already read-only. - wr.logger.Infof("demote tablet %v", tabletAliasStr) - masterStatus, err := wr.tmc.DemoteMaster(stopAllCtx, tablet) - if err != nil { - rec.RecordError(vterrors.Wrapf(err, "DemoteMaster failed on contested master %v", tabletAliasStr)) - return - } - pos, err := mysql.DecodePosition(masterStatus.Position) - if err != nil { - rec.RecordError(vterrors.Wrapf(err, "can't decode replication position for tablet %v", tabletAliasStr)) - return - } - positions <- tabletPos{ - tabletAliasStr: tabletAliasStr, - tablet: tablet, - pos: pos, - } - }(tabletAliasStr, tablet.Tablet) - } - wgStopAll.Wait() - close(positions) - if rec.HasErrors() { - return vterrors.Wrap(rec.Error(), "failed to demote all tablets") - } - - // Make a map of tablet positions. - tabletPosMap := make(map[string]tabletPos, len(tabletMap)) - for tp := range positions { - tabletPosMap[tp.tabletAliasStr] = tp - } - - // Make sure no tablet has a replication position farther ahead than the - // candidate master. It's up to our caller to choose a suitable - // candidate, and to choose another one if this check fails. - // - // Note that we still allow replication to run during this time, but we - // assume that no new high water mark can appear because we demoted all - // tablets to read-only. - // - // TODO: Consider temporarily replicating from another tablet to catch up. - tp, ok := tabletPosMap[masterElectTabletAliasStr] - if !ok { - return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "master-elect tablet %v not found in tablet map", masterElectTabletAliasStr) - } - masterElectPos := tp.pos - for _, tp := range tabletPosMap { - // The master elect pos has to be at least as far as every tablet. - if !masterElectPos.AtLeast(tp.pos) { - return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "tablet %v position (%v) contains transactions not found in master-elect %v position (%v)", - tp.tabletAliasStr, tp.pos, masterElectTabletAliasStr, masterElectPos) - } - } - - // Check we still have the topology lock. - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrap(err, "lost topology lock; aborting") - } - - // Promote the selected candidate to master. - promoteCtx, promoteCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer promoteCancel() - rp, err := wr.tmc.PromoteReplica(promoteCtx, masterElectTabletInfo.Tablet) - if err != nil { - return vterrors.Wrapf(err, "failed to promote %v to master", masterElectTabletAliasStr) - } - reparentJournalPos = rp - } else if topoproto.TabletAliasEqual(currentMaster.Alias, masterElectTabletAlias) { - // It is possible that a previous attempt to reparent failed to SetReadWrite - // so call it here to make sure underlying mysql is ReadWrite - rwCtx, rwCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer rwCancel() - - if err := wr.tmc.SetReadWrite(rwCtx, masterElectTabletInfo.Tablet); err != nil { - return vterrors.Wrapf(err, "failed to SetReadWrite on current master %v", masterElectTabletAliasStr) - } - // The master is already the one we want according to its tablet record. - refreshCtx, refreshCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer refreshCancel() - - // Get the position so we can try to fix replicas (below). - rp, err := wr.tmc.MasterPosition(refreshCtx, masterElectTabletInfo.Tablet) - if err != nil { - return vterrors.Wrapf(err, "failed to get replication position of current master %v", masterElectTabletAliasStr) - } - reparentJournalPos = rp - } else { - // There is already a master and it's not the one we want. - oldMasterTabletInfo := currentMaster - ev.OldMaster = *oldMasterTabletInfo.Tablet - - // Before demoting the old master, first make sure replication is - // working from the old master to the candidate master. If it's not - // working, we can't do a planned reparent because the candidate won't - // catch up. - wr.logger.Infof("Checking replication on master-elect %v", masterElectTabletAliasStr) - - // First we find the position of the current master. Note that this is - // just a snapshot of the position since we let it keep accepting new - // writes until we're sure we're going to proceed. - snapshotCtx, snapshotCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer snapshotCancel() - - snapshotPos, err := wr.tmc.MasterPosition(snapshotCtx, currentMaster.Tablet) - if err != nil { - return vterrors.Wrapf(err, "can't get replication position on current master %v; current master must be healthy to perform planned reparent", currentMaster.AliasString()) - } - - // Now wait for the master-elect to catch up to that snapshot point. - // If it catches up to that point within the waitReplicasTimeout, - // we can be fairly confident it will catch up on everything that's - // happened in the meantime once we demote the master to stop writes. - // - // We do this as an idempotent SetMaster to make sure the replica knows - // who the current master is. - setMasterCtx, setMasterCancel := context.WithTimeout(ctx, waitReplicasTimeout) - defer setMasterCancel() - - err = wr.tmc.SetMaster(setMasterCtx, masterElectTabletInfo.Tablet, currentMaster.Alias, 0, snapshotPos, true) - if err != nil { - return vterrors.Wrapf(err, "replication on master-elect %v did not catch up in time; replication must be healthy to perform planned reparent", masterElectTabletAliasStr) - } - - // Check we still have the topology lock. - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrap(err, "lost topology lock; aborting") - } - - // Demote the old master and get its replication position. It's fine if - // the old master was already demoted, since DemoteMaster is idempotent. - wr.logger.Infof("demote current master %v", oldMasterTabletInfo.Alias) - event.DispatchUpdate(ev, "demoting old master") - - demoteCtx, demoteCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer demoteCancel() - - masterStatus, err := wr.tmc.DemoteMaster(demoteCtx, oldMasterTabletInfo.Tablet) - if err != nil { - return fmt.Errorf("old master tablet %v DemoteMaster failed: %v", topoproto.TabletAliasString(shardInfo.MasterAlias), err) - } - - waitCtx, waitCancel := context.WithTimeout(ctx, waitReplicasTimeout) - defer waitCancel() - - waitErr := wr.tmc.WaitForPosition(waitCtx, masterElectTabletInfo.Tablet, masterStatus.Position) - if waitErr != nil || ctx.Err() == context.DeadlineExceeded { - // If the new master fails to catch up within the timeout, - // we try to roll back to the original master before aborting. - // It is possible that we have used up the original context, or that - // not enough time is left on it before it times out. - // But at this point we really need to be able to Undo so as not to - // leave the cluster in a bad state. - // So we create a fresh context based on context.Background(). - undoCtx, undoCancel := context.WithTimeout(context.Background(), *topo.RemoteOperationTimeout) - defer undoCancel() - if undoErr := wr.tmc.UndoDemoteMaster(undoCtx, oldMasterTabletInfo.Tablet); undoErr != nil { - log.Warningf("Encountered error while trying to undo DemoteMaster: %v", undoErr) - } - if waitErr != nil { - return vterrors.Wrapf(err, "master-elect tablet %v failed to catch up with replication", masterElectTabletAliasStr) - } - return vterrors.New(vtrpcpb.Code_DEADLINE_EXCEEDED, "PlannedReparent timed out, please try again.") - } - - promoteCtx, promoteCancel := context.WithTimeout(ctx, waitReplicasTimeout) - defer promoteCancel() - rp, err := wr.tmc.PromoteReplica(promoteCtx, masterElectTabletInfo.Tablet) - if err != nil { - return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master - please try again", masterElectTabletAliasStr) - } - - if ctx.Err() == context.DeadlineExceeded { - // PromoteReplica succeeded but the context has expired. PRS needs to be re-run to complete - return vterrors.New(vtrpcpb.Code_DEADLINE_EXCEEDED, "PlannedReparent timed out after promoting new master. Please re-run to fixup replicas.") - } - reparentJournalPos = rp - } - - // Check we still have the topology lock. - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrap(err, "lost topology lock, aborting") - } - - // Create a cancelable context for the following RPCs. - // If error conditions happen, we can cancel all outgoing RPCs. - replCtx, replCancel := context.WithTimeout(ctx, waitReplicasTimeout) - defer replCancel() - - // Go through all the tablets: - // - new master: populate the reparent journal - // - everybody else: reparent to new master, wait for row - event.DispatchUpdate(ev, "reparenting all tablets") - - // We add a (hopefully) unique record to the reparent journal table on the - // new master so we can check if replicas got it through replication. - reparentJournalTimestamp := time.Now().UnixNano() - - // Point all replicas at the new master and check that they receive the - // reparent journal entry, proving they are replicating from the new master. - // We do this concurrently with adding the journal entry (below), because - // if semi-sync is enabled, the update to the journal table can't succeed - // until at least one replica is successfully attached to the new master. - wgReplicas := sync.WaitGroup{} - rec := concurrency.AllErrorRecorder{} - for alias, tabletInfo := range tabletMap { - if alias == masterElectTabletAliasStr { - continue - } - wgReplicas.Add(1) - go func(alias string, tabletInfo *topo.TabletInfo) { - defer wgReplicas.Done() - wr.logger.Infof("setting new master on replica %v", alias) - - // We used to force replica start on the old master, but now that - // we support "resuming" a PRS attempt that failed, we can no - // longer assume that we know who the old master was. - // Instead, we rely on the old master to remember that it needs - // to start replication after being converted to a replica. - forceStartReplication := false - - if err := wr.tmc.SetMaster(replCtx, tabletInfo.Tablet, masterElectTabletAlias, reparentJournalTimestamp, "", forceStartReplication); err != nil { - rec.RecordError(fmt.Errorf("tablet %v SetMaster failed: %v", alias, err)) - return - } - }(alias, tabletInfo) - } - - // Add a reparent journal entry on the new master. - wr.logger.Infof("populating reparent journal on new master %v", masterElectTabletAliasStr) - err = wr.tmc.PopulateReparentJournal(replCtx, masterElectTabletInfo.Tablet, reparentJournalTimestamp, plannedReparentShardOperation, masterElectTabletAlias, reparentJournalPos) - if err != nil { - // The master failed. There's no way the replicas will work, so cancel them all. - wr.logger.Warningf("master failed to PopulateReparentJournal, canceling replica reparent attempts") - replCancel() - wgReplicas.Wait() - return fmt.Errorf("failed to PopulateReparentJournal on master: %v", err) - } - - // Wait for the replicas to complete. - wgReplicas.Wait() - if err := rec.Error(); err != nil { - wr.Logger().Errorf2(err, "some replicas failed to reparent; retry PlannedReparentShard with the same new master alias to retry failed replicas") - return err - } - - return nil -} - // EmergencyReparentShard will make the provided tablet the master for // the shard, when the old master is completely unreachable. func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard string, masterElectTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration, ignoredTablets sets.String) (err error) { From 1079a4a0362eca820bfab00a3bb2ded57f6055cc Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 Feb 2021 15:49:07 -0500 Subject: [PATCH 5/7] Bugfix: `SetMaster` call should not specify a wait position Signed-off-by: Andrew Mason --- go/vt/vtctl/reparentutil/planned_reparenter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go index 3873984d9e7..fe0bdcffedf 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -592,7 +592,7 @@ func (pr *PlannedReparenter) reparentTablets( // that it needs to start replication after transitioning from // MASTER => REPLICA. forceStartReplication := false - if err := pr.tmc.SetMaster(replCtx, tablet, ev.NewMaster.Alias, reparentJournalTimestamp, reparentJournalPosition, forceStartReplication); err != nil { + if err := pr.tmc.SetMaster(replCtx, tablet, ev.NewMaster.Alias, reparentJournalTimestamp, "", forceStartReplication); err != nil { rec.RecordError(vterrors.Wrapf(err, "tablet %v failed SetMaster(%v): %v", alias, primaryElectAliasStr, err)) } }(alias, tabletInfo.Tablet) From 1554187b5dcde5da20df5765a39164af6fc33152 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 Feb 2021 15:49:28 -0500 Subject: [PATCH 6/7] Update error substrings used in PRS test assertions Signed-off-by: Andrew Mason --- go/test/endtoend/reparent/reparent_test.go | 6 +++--- go/vt/vtctl/reparentutil/planned_reparenter.go | 2 +- go/vt/wrangler/testlib/planned_reparent_shard_test.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go/test/endtoend/reparent/reparent_test.go b/go/test/endtoend/reparent/reparent_test.go index c8f0719dbb9..5884a62e48a 100644 --- a/go/test/endtoend/reparent/reparent_test.go +++ b/go/test/endtoend/reparent/reparent_test.go @@ -204,7 +204,7 @@ func TestReparentReplicaOffline(t *testing.T) { // Perform a graceful reparent operation. out, err := prsWithTimeout(t, tab2, false, "", "31s") require.Error(t, err) - assert.Contains(t, out, fmt.Sprintf("tablet %s SetMaster failed", tab4.Alias)) + assert.Contains(t, out, fmt.Sprintf("tablet %s failed to SetMaster", tab4.Alias)) checkMasterTablet(t, tab2) } @@ -345,7 +345,7 @@ func TestReparentWithDownReplica(t *testing.T) { // Perform a graceful reparent operation. It will fail as one tablet is down. out, err := prs(t, tab2) require.Error(t, err) - assert.Contains(t, out, fmt.Sprintf("tablet %s SetMaster failed", tab3.Alias)) + assert.Contains(t, out, fmt.Sprintf("tablet %s failed to SetMaster", tab3.Alias)) // insert data into the new master, check the connected replica work confirmReplication(t, tab2, []*cluster.Vttablet{tab1, tab4}) @@ -441,5 +441,5 @@ func TestReparentDoesntHangIfMasterFails(t *testing.T) { // insert. The replicas should then abort right away. out, err := prs(t, tab2) require.Error(t, err) - assert.Contains(t, out, "master failed to PopulateReparentJournal") + assert.Contains(t, out, "primary failed to PopulateReparentJournal") } diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go index fe0bdcffedf..cee4587c8b4 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -593,7 +593,7 @@ func (pr *PlannedReparenter) reparentTablets( // MASTER => REPLICA. forceStartReplication := false if err := pr.tmc.SetMaster(replCtx, tablet, ev.NewMaster.Alias, reparentJournalTimestamp, "", forceStartReplication); err != nil { - rec.RecordError(vterrors.Wrapf(err, "tablet %v failed SetMaster(%v): %v", alias, primaryElectAliasStr, err)) + rec.RecordError(vterrors.Wrapf(err, "tablet %v failed to SetMaster(%v): %v", alias, primaryElectAliasStr, err)) } }(alias, tabletInfo.Tablet) } diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 59f10c4dc26..e84a1043311 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -290,7 +290,7 @@ func TestPlannedReparentNoMaster(t *testing.T) { err := vp.Run([]string{"PlannedReparentShard", "-wait_replicas_timeout", "10s", "-keyspace_shard", replica1.Tablet.Keyspace + "/" + replica1.Tablet.Shard, "-new_master", topoproto.TabletAliasString(replica1.Tablet.Alias)}) assert.Error(t, err) - assert.Contains(t, err.Error(), "the shard has no master") + assert.Contains(t, err.Error(), "the shard has no current primary") } // TestPlannedReparentShardWaitForPositionFail simulates a failure of the WaitForPosition call @@ -386,7 +386,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_replicas_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) assert.Error(t, err) - assert.Contains(t, err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") + assert.Contains(t, err.Error(), "replication on primary-elect cell1-0000000001 did not catch up in time") // now check that DemoteMaster was undone and old master is still master assert.True(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly not set") @@ -486,7 +486,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_replicas_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) assert.Error(t, err) - assert.Contains(t, err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") + assert.Contains(t, err.Error(), "replication on primary-elect cell1-0000000001 did not catch up in time") // now check that DemoteMaster was undone and old master is still master assert.True(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly not set") From f6913d43dbb63ac5f6a9fc60488bd71616ae61b8 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 17 Feb 2021 20:53:49 -0500 Subject: [PATCH 7/7] Fix typo in comments Signed-off-by: Andrew Mason --- go/vt/vtctl/reparentutil/planned_reparenter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go index cee4587c8b4..100d1e3d4a1 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -125,7 +125,7 @@ func (pr *PlannedReparenter) getLockAction(opts PlannedReparentOptions) string { ) } -// prelightChecks checks some invariants that pr.reparentShardLocked() depends +// preflightChecks checks some invariants that pr.reparentShardLocked() depends // on. It returns a boolean to indicate if the reparent is a no-op (which // happens iff the caller specified an AvoidPrimaryAlias and it's not the shard // primary), as well as an error.