From d4bf90d57332b84a096e90d3c8e6f1740c256c91 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 8 Jan 2025 13:19:45 -0800 Subject: [PATCH] for remote commands make using SSH control master optional --- cmd/metrics/metadata.go | 4 +-- cmd/metrics/nmi_watchdog.go | 6 ++-- cmd/metrics/process.go | 6 ++-- internal/script/script.go | 6 ++-- internal/target/target.go | 68 ++++++++++++++++++++++--------------- 5 files changed, 51 insertions(+), 39 deletions(-) diff --git a/cmd/metrics/metadata.go b/cmd/metrics/metadata.go index 068fe9f..4768a3a 100644 --- a/cmd/metrics/metadata.go +++ b/cmd/metrics/metadata.go @@ -343,7 +343,7 @@ func getUncoreDeviceIDs(myTarget target.Target, localTempDir string) (IDs map[st // getCPUInfo - reads and returns all data from /proc/cpuinfo func getCPUInfo(myTarget target.Target) (cpuInfo []map[string]string, err error) { cmd := exec.Command("cat", "/proc/cpuinfo") - stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0) + stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true) if err != nil { err = fmt.Errorf("failed to get cpuinfo: %s, %d, %v", stderr, exitcode, err) return @@ -369,7 +369,7 @@ func getCPUInfo(myTarget target.Target) (cpuInfo []map[string]string, err error) // 'perf list' func getPerfSupportedEvents(myTarget target.Target, perfPath string) (supportedEvents string, err error) { cmd := exec.Command(perfPath, "list") - stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0) + stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true) if err != nil { err = fmt.Errorf("failed to get perf list: %s, %d, %v", stderr, exitcode, err) return diff --git a/cmd/metrics/nmi_watchdog.go b/cmd/metrics/nmi_watchdog.go index b095a10..7fe9115 100644 --- a/cmd/metrics/nmi_watchdog.go +++ b/cmd/metrics/nmi_watchdog.go @@ -48,7 +48,7 @@ func getNMIWatchdog(myTarget target.Target) (setting string, err error) { return } cmd := exec.Command(sysctl, "kernel.nmi_watchdog") - stdout, _, _, err := myTarget.RunCommand(cmd, 0) + stdout, _, _, err := myTarget.RunCommand(cmd, 0, true) if err != nil { return } @@ -86,7 +86,7 @@ func setNMIWatchdog(myTarget target.Target, setting string, localTempDir string) // findSysctl - gets a useable path to sysctl or error func findSysctl(myTarget target.Target) (path string, err error) { cmd := exec.Command("which", "sysctl") - stdout, _, _, err := myTarget.RunCommand(cmd, 0) + stdout, _, _, err := myTarget.RunCommand(cmd, 0, true) if err == nil { //found it path = strings.TrimSpace(stdout) @@ -95,7 +95,7 @@ func findSysctl(myTarget target.Target) (path string, err error) { // didn't find it on the path, try being specific sbinPath := "/usr/sbin/sysctl" cmd = exec.Command("which", sbinPath) - _, _, _, err = myTarget.RunCommand(cmd, 0) + _, _, _, err = myTarget.RunCommand(cmd, 0, true) if err == nil { // found it path = sbinPath diff --git a/cmd/metrics/process.go b/cmd/metrics/process.go index 8f718c2..6a0fa96 100644 --- a/cmd/metrics/process.go +++ b/cmd/metrics/process.go @@ -63,7 +63,7 @@ func GetCgroups(myTarget target.Target, cids []string, localTempDir string) (cgr func GetHotProcesses(myTarget target.Target, maxProcesses int, filter string) (processes []Process, err error) { // run ps to get list of processes sorted by cpu utilization (descending) cmd := exec.Command("ps", "-a", "-x", "-h", "-o", "pid,ppid,comm,cmd", "--sort=-%cpu") - stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0) + stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true) if err != nil { err = fmt.Errorf("failed to get hot processes: %s, %d, %v", stderr, exitcode, err) return @@ -173,7 +173,7 @@ done | sort -nr | head -n %d func processExists(myTarget target.Target, pid string) (exists bool) { cmd := exec.Command("ps", "-p", pid) - _, _, _, err := myTarget.RunCommand(cmd, 0) + _, _, _, err := myTarget.RunCommand(cmd, 0, true) if err != nil { exists = false return @@ -184,7 +184,7 @@ func processExists(myTarget target.Target, pid string) (exists bool) { func getProcess(myTarget target.Target, pid string) (process Process, err error) { cmd := exec.Command("ps", "-q", pid, "h", "-o", "pid,ppid,comm,cmd", "ww") - stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0) + stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true) if err != nil { err = fmt.Errorf("failed to get process: %s, %d, %v", stderr, exitcode, err) return diff --git a/internal/script/script.go b/internal/script/script.go index 4f970c1..f290641 100644 --- a/internal/script/script.go +++ b/internal/script/script.go @@ -160,7 +160,7 @@ func RunScripts(myTarget target.Target, scripts []ScriptDefinition, ignoreScript } else { cmd = exec.Command("bash", path.Join(myTarget.GetTempDirectory(), masterScriptName)) } - stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0) + stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, false) // don't reuse ssh connection on long-running commands, makes it difficult to kill the command if err != nil { slog.Error("error running master script on target", slog.String("stdout", stdout), slog.String("stderr", stderr), slog.Int("exitcode", exitcode), slog.String("error", err.Error())) return nil, err @@ -183,7 +183,7 @@ func RunScripts(myTarget target.Target, scripts []ScriptDefinition, ignoreScript // run sequential scripts for _, script := range sequentialScripts { cmd := prepareCommand(script, myTarget.GetTempDirectory()) - stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0) + stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, false) if err != nil { slog.Error("error running script on target", slog.String("script", script.Script), slog.String("stdout", stdout), slog.String("stderr", stderr), slog.Int("exitcode", exitcode), slog.String("error", err.Error())) } @@ -241,7 +241,7 @@ func RunScriptAsync(myTarget target.Target, script ScriptDefinition, localTempDi }() } cmd := prepareCommand(script, myTarget.GetTempDirectory()) - err = myTarget.RunCommandAsync(cmd, stdoutChannel, stderrChannel, exitcodeChannel, 0, cmdChannel) + err = myTarget.RunCommandAsync(cmd, 0, false, stdoutChannel, stderrChannel, exitcodeChannel, cmdChannel) errorChannel <- err } diff --git a/internal/target/target.go b/internal/target/target.go index 206d8d0..863a967 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -60,12 +60,24 @@ type Target interface { GetUserPath() (path string, err error) // RunCommand runs the specified command on the target. + // Arguments: + // - cmd: the command to run + // - timeout: the maximum time allowed for the command to run (zero means no timeout) + // - reuseSSHConnection: whether to reuse the SSH connection for the command (only relevant for RemoteTarget) // It returns the standard output, standard error, exit code, and any error that occurred. - RunCommand(cmd *exec.Cmd, timeout int) (stdout string, stderr string, exitCode int, err error) + RunCommand(cmd *exec.Cmd, timeout int, reuseSSHConnection bool) (stdout string, stderr string, exitCode int, err error) // RunCommandAsync runs the specified command on the target in an asynchronous manner. + // Arguments: + // - cmd: the command to run + // - timeout: the maximum time allowed for the command to run (zero means no timeout) + // - reuseSSHConnection: whether to reuse the SSH connection for the command (only relevant for RemoteTarget) + // - stdoutChannel: a channel to send the standard output of the command + // - stderrChannel: a channel to send the standard error of the command + // - exitcodeChannel: a channel to send the exit code of the command + // - cmdChannel: a channel to send the command that was run // It returns any error that occurred. - RunCommandAsync(cmd *exec.Cmd, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, timeout int, cmdChannel chan *exec.Cmd) error + RunCommandAsync(cmd *exec.Cmd, timeout int, reuseSSHConnection bool, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, cmdChannel chan *exec.Cmd) error // PushFile transfers a file from the local system to the target. // It returns any error that occurred. @@ -171,7 +183,7 @@ func (t *RemoteTarget) SetSshPass(sshPass string) { // RunCommand executes the given command with a timeout and returns the standard output, // standard error, exit code, and any error that occurred. -func (t *LocalTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, stderr string, exitCode int, err error) { +func (t *LocalTarget) RunCommand(cmd *exec.Cmd, timeout int, argNotUsed bool) (stdout string, stderr string, exitCode int, err error) { input := "" if t.sudo != "" && len(cmd.Args) > 2 && cmd.Args[0] == "sudo" && strings.HasPrefix(cmd.Args[1], "-") && strings.Contains(cmd.Args[1], "S") { // 'sudo -S' gets password from stdin input = t.sudo + "\n" @@ -179,8 +191,8 @@ func (t *LocalTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, std return runLocalCommandWithInputWithTimeout(cmd, input, timeout) } -func (t *RemoteTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, stderr string, exitCode int, err error) { - localCommand := t.prepareLocalCommand(cmd, false) +func (t *RemoteTarget) RunCommand(cmd *exec.Cmd, timeout int, reuseSSHConnection bool) (stdout string, stderr string, exitCode int, err error) { + localCommand := t.prepareLocalCommand(cmd, reuseSSHConnection) return runLocalCommandWithInputWithTimeout(localCommand, "", timeout) } @@ -190,15 +202,15 @@ func (t *RemoteTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, st // and the exit code is sent to the exitcodeChannel. // The timeout parameter specifies the maximum time allowed for the command to run. // Returns an error if there was a problem running the command. -func (t *LocalTarget) RunCommandAsync(cmd *exec.Cmd, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, timeout int, cmdChannel chan *exec.Cmd) (err error) { +func (t *LocalTarget) RunCommandAsync(cmd *exec.Cmd, timeout int, argNotUsed bool, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, cmdChannel chan *exec.Cmd) (err error) { localCommand := cmd cmdChannel <- localCommand err = runLocalCommandWithInputWithTimeoutAsync(localCommand, stdoutChannel, stderrChannel, exitcodeChannel, "", timeout) return } -func (t *RemoteTarget) RunCommandAsync(cmd *exec.Cmd, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, timeout int, cmdChannel chan *exec.Cmd) (err error) { - localCommand := t.prepareLocalCommand(cmd, true) +func (t *RemoteTarget) RunCommandAsync(cmd *exec.Cmd, timeout int, reuseSSHConnection bool, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, cmdChannel chan *exec.Cmd) (err error) { + localCommand := t.prepareLocalCommand(cmd, reuseSSHConnection) cmdChannel <- localCommand err = runLocalCommandWithInputWithTimeoutAsync(localCommand, stdoutChannel, stderrChannel, exitcodeChannel, "", timeout) return @@ -266,7 +278,7 @@ func (t *RemoteTarget) CreateTempDirectory(rootDir string) (tempDir string, err root = fmt.Sprintf("--tmpdir=%s", rootDir) } cmd := exec.Command("mktemp", "-d", "-t", root, "perfspect.tmp.XXXXXXXXXX", "|", "xargs", "realpath") - tempDir, _, _, err = t.RunCommand(cmd, 0) + tempDir, _, _, err = t.RunCommand(cmd, 0, true) tempDir = strings.TrimSpace(tempDir) t.tempDir = tempDir return @@ -331,7 +343,7 @@ func (t *LocalTarget) CreateDirectory(baseDir string, targetDir string) (dir str func (t *RemoteTarget) CreateDirectory(baseDir string, targetDir string) (dir string, err error) { dir = filepath.Join(baseDir, targetDir) cmd := exec.Command("mkdir", dir) - _, _, _, err = t.RunCommand(cmd, 0) + _, _, _, err = t.RunCommand(cmd, 0, true) return } @@ -348,7 +360,7 @@ func (t *LocalTarget) RemoveDirectory(targetDir string) (err error) { func (t *RemoteTarget) RemoveDirectory(targetDir string) (err error) { if targetDir != "" { cmd := exec.Command("rm", "-rf", targetDir) - _, _, _, err = t.RunCommand(cmd, 0) + _, _, _, err = t.RunCommand(cmd, 0, true) } return } @@ -360,7 +372,7 @@ func (t *LocalTarget) CanConnect() bool { func (t *RemoteTarget) CanConnect() bool { cmd := exec.Command("exit", "0") - _, _, _, err := t.RunCommand(cmd, 5) + _, _, _, err := t.RunCommand(cmd, 5, true) return err == nil } @@ -388,14 +400,14 @@ func (t *LocalTarget) CanElevatePrivileges() bool { slog.Error("error writing sudo password", slog.String("error", err.Error())) } }() - _, _, _, err := t.RunCommand(cmd, 0) + _, _, _, err := t.RunCommand(cmd, 0, true) if err == nil { t.canElevate = 1 return true // sudo password works } } cmd := exec.Command("sudo", "-kS", "ls") - _, _, _, err := t.RunCommand(cmd, 0) + _, _, _, err := t.RunCommand(cmd, 0, true) if err == nil { // true - passwordless sudo works t.canElevate = 1 return true @@ -415,7 +427,7 @@ func (t *RemoteTarget) CanElevatePrivileges() bool { return true } cmd := exec.Command("sudo", "-kS", "ls") - _, _, _, err := t.RunCommand(cmd, 0) + _, _, _, err := t.RunCommand(cmd, 0, true) if err == nil { // true - passwordless sudo works t.canElevate = 1 return true @@ -492,7 +504,7 @@ func (t *LocalTarget) GetUserPath() (string, error) { func (t *RemoteTarget) GetUserPath() (string, error) { if t.userPath == "" { cmd := exec.Command("echo", "$PATH") - stdout, _, _, err := t.RunCommand(cmd, 0) + stdout, _, _, err := t.RunCommand(cmd, 0, true) if err != nil { return "", err } @@ -594,7 +606,7 @@ func runLocalCommandWithInputWithTimeoutAsync(cmd *exec.Cmd, stdoutChannel chan return nil } -func (t *RemoteTarget) prepareSSHFlags(scp bool, async bool, prompt bool) (flags []string) { +func (t *RemoteTarget) prepareSSHFlags(scp bool, useControlMaster bool, prompt bool) (flags []string) { flags = []string{ "-2", "-o", @@ -621,7 +633,7 @@ func (t *RemoteTarget) prepareSSHFlags(scp bool, async bool, prompt bool) (flags flags = append(flags, promptFlags...) } // when using a control master, a long-running remote program doesn't get terminated when the local ssh client is terminated - if !async { + if useControlMaster { controlPathFlags := []string{ "-o", "ControlPath=" + filepath.Join(os.TempDir(), `control-%h-%p-%r`), @@ -654,10 +666,10 @@ func (t *RemoteTarget) prepareSSHFlags(scp bool, async bool, prompt bool) (flags return } -func (t *RemoteTarget) prepareSSHCommand(command []string, async bool, prompt bool) []string { +func (t *RemoteTarget) prepareSSHCommand(command []string, useControlMaster bool, prompt bool) []string { var cmd []string cmd = append(cmd, "ssh") - cmd = append(cmd, t.prepareSSHFlags(false, async, prompt)...) + cmd = append(cmd, t.prepareSSHFlags(false, useControlMaster, prompt)...) if t.user != "" { cmd = append(cmd, t.user+"@"+t.host) } else { @@ -671,7 +683,7 @@ func (t *RemoteTarget) prepareSSHCommand(command []string, async bool, prompt bo func (t *RemoteTarget) prepareSCPCommand(src string, dstDir string, push bool) []string { var cmd []string cmd = append(cmd, "scp") - cmd = append(cmd, t.prepareSSHFlags(true, false, false)...) + cmd = append(cmd, t.prepareSSHFlags(true, true, false)...) if push { fileInfo, err := os.Stat(src) if err != nil { @@ -698,11 +710,11 @@ func (t *RemoteTarget) prepareSCPCommand(src string, dstDir string, push bool) [ return cmd } -func (t *RemoteTarget) prepareLocalCommand(cmd *exec.Cmd, async bool) *exec.Cmd { +func (t *RemoteTarget) prepareLocalCommand(cmd *exec.Cmd, useControlMaster bool) *exec.Cmd { var name string var args []string usePass := t.key == "" && t.sshPass != "" - sshCommand := t.prepareSSHCommand(cmd.Args, async, usePass) + sshCommand := t.prepareSSHCommand(cmd.Args, useControlMaster, usePass) if usePass { name = t.sshpassPath args = []string{"-e", "--"} @@ -741,7 +753,7 @@ func (t *RemoteTarget) prepareAndRunSCPCommand(srcPath string, dstDir string, is func getArchitecture(t Target) (arch string, err error) { cmd := exec.Command("uname", "-m") - arch, _, _, err = t.RunCommand(cmd, 0) + arch, _, _, err = t.RunCommand(cmd, 0, true) if err != nil { return } @@ -751,7 +763,7 @@ func getArchitecture(t Target) (arch string, err error) { func getFamily(t Target) (family string, err error) { cmd := exec.Command("bash", "-c", "lscpu | grep -i \"^CPU family:\" | awk '{print $NF}'") - family, _, _, err = t.RunCommand(cmd, 0) + family, _, _, err = t.RunCommand(cmd, 0, true) if err != nil { return } @@ -761,7 +773,7 @@ func getFamily(t Target) (family string, err error) { func getModel(t Target) (model string, err error) { cmd := exec.Command("bash", "-c", "lscpu | grep -i model: | awk '{print $NF}'") - model, _, _, err = t.RunCommand(cmd, 0) + model, _, _, err = t.RunCommand(cmd, 0, true) if err != nil { return } @@ -776,7 +788,7 @@ func installLkms(t Target, lkms []string) (installedLkms []string, err error) { } for _, lkm := range lkms { slog.Debug("attempting to install kernel module", slog.String("lkm", lkm)) - _, _, _, err := t.RunCommand(exec.Command("modprobe", "--first-time", lkm), 10) + _, _, _, err := t.RunCommand(exec.Command("modprobe", "--first-time", lkm), 10, true) if err != nil { slog.Debug("kernel module already installed or problem installing", slog.String("lkm", lkm), slog.String("error", err.Error())) continue @@ -794,7 +806,7 @@ func uninstallLkms(t Target, lkms []string) (err error) { } for _, lkm := range lkms { slog.Debug("attempting to uninstall kernel module", slog.String("lkm", lkm)) - _, _, _, err := t.RunCommand(exec.Command("modprobe", "-r", lkm), 10) + _, _, _, err := t.RunCommand(exec.Command("modprobe", "-r", lkm), 10, true) if err != nil { slog.Error("error uninstalling kernel module", slog.String("lkm", lkm), slog.String("error", err.Error())) continue