Skip to content

Commit

Permalink
replicaset: add rebootstrap command
Browse files Browse the repository at this point in the history
Closes #316

@TarantoolBot document
Title: `tt replicaset rebootstrap` re-bootstraps an instance

This patch adds new subcommand for the replicaset module.
`tt replicaset rebootstrap [flags] <APP_NAME:INSTANCE_NAME>`
re-bootstraps an instance: stops it, removes instance artifacts,
starts it again.
  • Loading branch information
psergee committed Jul 20, 2024
1 parent abb860d commit 86f1484
Show file tree
Hide file tree
Showing 12 changed files with 620 additions and 65 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `tt enable`: create a symbolic link in 'instances_enabled' directory to a script or
an application directory.
- `tt replicaset bootstrap`: command to bootstrap a Cartridge cluster or an instance.
- `tt rs rebootstrap`: re-bootstraps an instance.

### Fixed

Expand Down
45 changes: 45 additions & 0 deletions cli/cmd/replicaset.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ var (
replicasetBootstrapVshard bool
replicasetCartridgeReplicasetsFile string
replicasetReplicasetName string
rebootstrapConfirmed bool

replicasetUriHelp = " The URI can be specified in the following formats:\n" +
" * [tcp://][username:password@][host:port]\n" +
Expand Down Expand Up @@ -220,6 +221,27 @@ func newVShardCmd() *cobra.Command {
return cmd
}

// newRebootstrapCmd creates a "replicaset rebootstrap" command.
func newRebootstrapCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "rebootstrap <APP_NAME:INSTANCE_NAME>",
DisableFlagsInUseLine: true,
Short: "Re-bootstraps an instance",
Run: func(cmd *cobra.Command, args []string) {
cmdCtx.CommandName = cmd.Name()
err := modules.RunCmd(&cmdCtx, cmd.CommandPath(), &modulesInfo,
internalReplicasetRebootstrapModule, args)
util.HandleCmdErr(cmd, err)
},
Args: cobra.ExactArgs(1),
}

cmd.Flags().BoolVarP(&rebootstrapConfirmed, "yes", "y", false,
"automatically confirm rebootstrap")

return cmd
}

// NewReplicasetCmd creates a replicaset command.
func NewReplicasetCmd() *cobra.Command {
cmd := &cobra.Command{
Expand All @@ -234,6 +256,7 @@ func NewReplicasetCmd() *cobra.Command {
cmd.AddCommand(newExpelCmd())
cmd.AddCommand(newVShardCmd())
cmd.AddCommand(newBootstrapCmd())
cmd.AddCommand(newRebootstrapCmd())

return cmd
}
Expand Down Expand Up @@ -525,3 +548,25 @@ func getOrchestrator() (replicaset.Orchestrator, error) {
}
return orchestrator, nil
}

// internalReplicasetRebootstrapModule is a "rebootstrap" command for the replicaset module.
func internalReplicasetRebootstrapModule(cmdCtx *cmdcontext.CmdCtx, args []string) error {
if len(args) > 1 {
return util.NewArgError("only one instance supported for re-bootstrap")
}
if len(args) < 1 {
return util.NewArgError("instance for rebootstrap is not specified")
}

appName, instName, found := strings.Cut(args[0], string(running.InstanceDelimiter))
if !found {
return util.NewArgError(
"an instance name is not specified. Please use app:instance format.")
}

return replicaset.Rebootstrap(*cmdCtx, *cliOpts, replicaset.RebootstrapCtx{
AppName: appName,
InstanceName: instName,
Confirmed: rebootstrapConfirmed,
})
}
61 changes: 9 additions & 52 deletions cli/cmd/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,13 @@ package cmd
import (
"fmt"
"os"
"os/exec"
"strconv"
"syscall"
"time"

"github.com/apex/log"
"github.com/spf13/cobra"
"github.com/tarantool/tt/cli/cmd/internal"
"github.com/tarantool/tt/cli/cmdcontext"
"github.com/tarantool/tt/cli/modules"
"github.com/tarantool/tt/cli/process_utils"
"github.com/tarantool/tt/cli/running"
"github.com/tarantool/tt/cli/util"
"github.com/tarantool/tt/lib/integrity"
Expand Down Expand Up @@ -59,60 +55,21 @@ func NewStartCmd() *cobra.Command {
return startCmd
}

// startWatchdog starts tarantool instance with watchdog.
func startWatchdog(ttExecutable string, instance running.InstanceCtx) error {
appName := running.GetAppInstanceName(instance)
// If an instance is already running don't try to start it again.
// For restarting an instance use tt restart command.
procStatus := process_utils.ProcessStatus(instance.PIDFile)
if procStatus.Code == process_utils.ProcStateRunning.Code {
log.Infof("The instance %s (PID = %d) is already running.", appName, procStatus.PID)
return nil
}

newArgs := []string{}
if cmdCtx.Cli.IntegrityCheck != "" {
newArgs = append(newArgs, "--integrity-check", cmdCtx.Cli.IntegrityCheck)
}

if cmdCtx.Cli.IsSystem {
newArgs = append(newArgs, "-S")
} else if cmdCtx.Cli.LocalLaunchDir != "" {
newArgs = append(newArgs, "-L", cmdCtx.Cli.LocalLaunchDir)
} else {
newArgs = append(newArgs, "--cfg", cmdCtx.Cli.ConfigPath)
}

newArgs = append(newArgs, "start", "--watchdog", appName)

if cmdCtx.Cli.IntegrityCheck != "" {
newArgs = append(newArgs, "--integrity-check-period",
strconv.Itoa(integrityCheckPeriod))
}

f, err := cmdCtx.Integrity.Repository.Read(ttExecutable)
if err != nil {
return err
}
f.Close()

log.Infof("Starting an instance [%s]...", appName)

wdCmd := exec.Command(ttExecutable, newArgs...)
// Set new pgid for watchdog process, so it will not be killed after a session is closed.
wdCmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
return wdCmd.Start()
}

// startInstancesUnderWatchdog starts tarantool instances under tt watchdog.
func startInstancesUnderWatchdog(instances []running.InstanceCtx) error {
func startInstancesUnderWatchdog(cmdCtx *cmdcontext.CmdCtx, instances []running.InstanceCtx) error {
ttBin, err := os.Executable()
if err != nil {
return err
}

startArgs := []string{}
if cmdCtx.Cli.IntegrityCheck != "" {
startArgs = append(startArgs, "--integrity-check-period",
strconv.Itoa(integrityCheckPeriod))
}

for _, instance := range instances {
if err := startWatchdog(ttBin, instance); err != nil {
if err := running.StartWatchdog(cmdCtx, ttBin, instance, startArgs); err != nil {
return err
}
}
Expand Down Expand Up @@ -140,7 +97,7 @@ func internalStartModule(cmdCtx *cmdcontext.CmdCtx, args []string) error {
}

if !watchdog {
if err := startInstancesUnderWatchdog(runningCtx.Instances); err != nil {
if err := startInstancesUnderWatchdog(cmdCtx, runningCtx.Instances); err != nil {
return err
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions cli/process_utils/process_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (procState ProcessState) String() string {
// GetPIDFromFile returns PID from the PIDFile.
func GetPIDFromFile(pidFileName string) (int, error) {
if _, err := os.Stat(pidFileName); err != nil {
return 0, fmt.Errorf(`can't "stat" the PID file. Error: "%v"`, err)
return 0, fmt.Errorf(`can't "stat" the PID file. Error: %w`, err)
}

pidFile, err := os.Open(pidFileName)
Expand Down Expand Up @@ -162,7 +162,7 @@ func getRunningPid(pidFile string) (int, error) {
func StopProcess(pidFile string) (int, error) {
pid, err := getRunningPid(pidFile)
if err != nil {
return 0, fmt.Errorf("can't get pid of running process: %s", err)
return 0, fmt.Errorf("can't get pid of running process: %w", err)
}

if err = syscall.Kill(pid, syscall.SIGINT); err != nil {
Expand Down
111 changes: 111 additions & 0 deletions cli/replicaset/rebootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package replicaset

import (
"fmt"
"os"
"path/filepath"

"github.com/apex/log"
"github.com/tarantool/tt/cli/cmdcontext"
"github.com/tarantool/tt/cli/config"
"github.com/tarantool/tt/cli/running"
"github.com/tarantool/tt/cli/util"
)

// RebootstrapCtx contains rebootstrap operations arguments.
type RebootstrapCtx struct {
// AppName is an application name the instance belongs to.
AppName string
// InstanceName is an instance name to re-bootstrap.
InstanceName string
// Confirmed is true if re-bootstrap confirmation is not required.
Confirmed bool
}

// cleanDataFiles removes snap, xlog and vinyl artifacts.
func cleanDataFiles(instCtx running.InstanceCtx) error {
filesToRemove := []string{}
for _, pattern := range [...]string{
filepath.Join(instCtx.MemtxDir, "*.snap"),
filepath.Join(instCtx.WalDir, "*.xlog"),
filepath.Join(instCtx.VinylDir, "*.vylog"),
} {
if foundFiles, err := filepath.Glob(pattern); err != nil {
return err
} else {
filesToRemove = append(filesToRemove, foundFiles...)
}
}

for _, fileToRemove := range filesToRemove {
stat, err := os.Stat(fileToRemove)
if err != nil {
if os.IsNotExist(err) {
continue
}
return fmt.Errorf("cannot get info of %q: %s", fileToRemove, err)
}
if stat.Mode().IsRegular() {
if err = os.Remove(fileToRemove); err != nil {
return fmt.Errorf("cannot remove %q: %s", fileToRemove, err)
}
log.Debugf("Removed %q", fileToRemove)
}
}

return nil
}

// Rebootstrap re-bootstraps the instance by stopping it, removing its artifacts,
// and starting it again.
func Rebootstrap(cmdCtx cmdcontext.CmdCtx, cliOpts config.CliOpts, rbCtx RebootstrapCtx) error {
apps, err := running.CollectInstancesForApps([]string{rbCtx.AppName}, &cliOpts,
cmdCtx.Cli.ConfigDir, cmdCtx.Integrity)
if err != nil {
return fmt.Errorf("cannot collect application instances info: %s", err)
}

found := false
var instCtx running.InstanceCtx
for _, instCtx = range apps[rbCtx.AppName] {
if instCtx.InstName == rbCtx.InstanceName {
found = true
break
}
}

if !found {
return fmt.Errorf("instance %q is not found", rbCtx.InstanceName)
}

if !rbCtx.Confirmed {
if yes, err := util.AskConfirm(os.Stdin, fmt.Sprintf(
"Rebootstrap will stop the instance %s and remove all its data files. "+
"Do you want to continue?",
rbCtx.InstanceName)); err != nil || !yes {
return err
}
}

log.Debugf("Stopping the instance")
if err = running.Stop(&instCtx); err != nil {
return fmt.Errorf("failed to stop the instance %s: %s", rbCtx.InstanceName, err)
}

if err = cleanDataFiles(instCtx); err != nil {
return fmt.Errorf("failed to remove instance's artifacts: %s", err)
}

// TODO: need to support integrity check continuation on this start.
// tarantool/tt-ee#203
log.Debugf("Starting the instance")
ttBin, err := os.Executable()
if err != nil {
return err
}
if err = running.StartWatchdog(&cmdCtx, ttBin, instCtx, []string{}); err != nil {
return fmt.Errorf("failed to start the instance: %s", err)
}

return nil
}
36 changes: 36 additions & 0 deletions cli/replicaset/rebootstrap_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package replicaset

import (
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tarantool/tt/cli/running"
)

func Test_cleanDataFiles(t *testing.T) {
tmpDir := t.TempDir()

require.NoError(t, os.WriteFile(
filepath.Join(tmpDir, "000.xlog"), []byte{}, 0755))
require.NoError(t, os.WriteFile(
filepath.Join(tmpDir, "000.snap"), []byte{}, 0755))
require.NoError(t, os.WriteFile(
filepath.Join(tmpDir, "000.vylog"), []byte{}, 0755))
require.NoError(t, os.WriteFile(
filepath.Join(tmpDir, "000.txt"), []byte{}, 0755))
require.NoError(t, os.Mkdir(
filepath.Join(tmpDir, "dir.snap"), 0755))

require.NoError(t, cleanDataFiles(running.InstanceCtx{
WalDir: tmpDir,
MemtxDir: tmpDir,
VinylDir: tmpDir}))
assert.NoFileExists(t, filepath.Join(tmpDir, "000.xlog"))
assert.NoFileExists(t, filepath.Join(tmpDir, "000.snap"))
assert.NoFileExists(t, filepath.Join(tmpDir, "000.vylog"))
assert.FileExists(t, filepath.Join(tmpDir, "000.txt"))
assert.DirExists(t, filepath.Join(tmpDir, "dir.snap"))
}
Loading

0 comments on commit 86f1484

Please sign in to comment.