Skip to content

Commit

Permalink
Merge pull request #145 from intel/ctrlc
Browse files Browse the repository at this point in the history 
Enable interruption of PerfSpect with SIGINT (ctrl-c) when collecting data over SSH
  • Loading branch information
@harp-intel
harp-intel authored Jan 9, 2025
2 parents b1af182 + d4bf90d commit de2ff60
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 39 deletions.
4 changes: 2 additions & 2 deletions cmd/metrics/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ func getUncoreDeviceIDs(myTarget target.Target, localTempDir string) (IDs map[st
// getCPUInfo - reads and returns all data from /proc/cpuinfo
func getCPUInfo(myTarget target.Target) (cpuInfo []map[string]string, err error) {
cmd := exec.Command("cat", "/proc/cpuinfo")
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0)
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true)
if err != nil {
err = fmt.Errorf("failed to get cpuinfo: %s, %d, %v", stderr, exitcode, err)
return
Expand All @@ -369,7 +369,7 @@ func getCPUInfo(myTarget target.Target) (cpuInfo []map[string]string, err error)
// 'perf list'
func getPerfSupportedEvents(myTarget target.Target, perfPath string) (supportedEvents string, err error) {
cmd := exec.Command(perfPath, "list")
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0)
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true)
if err != nil {
err = fmt.Errorf("failed to get perf list: %s, %d, %v", stderr, exitcode, err)
return
Expand Down
6 changes: 3 additions & 3 deletions cmd/metrics/nmi_watchdog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func getNMIWatchdog(myTarget target.Target) (setting string, err error) {
return
}
cmd := exec.Command(sysctl, "kernel.nmi_watchdog")
stdout, _, _, err := myTarget.RunCommand(cmd, 0)
stdout, _, _, err := myTarget.RunCommand(cmd, 0, true)
if err != nil {
return
}
Expand Down Expand Up @@ -86,7 +86,7 @@ func setNMIWatchdog(myTarget target.Target, setting string, localTempDir string)
// findSysctl - gets a useable path to sysctl or error
func findSysctl(myTarget target.Target) (path string, err error) {
cmd := exec.Command("which", "sysctl")
stdout, _, _, err := myTarget.RunCommand(cmd, 0)
stdout, _, _, err := myTarget.RunCommand(cmd, 0, true)
if err == nil {
//found it
path = strings.TrimSpace(stdout)
Expand All @@ -95,7 +95,7 @@ func findSysctl(myTarget target.Target) (path string, err error) {
// didn't find it on the path, try being specific
sbinPath := "/usr/sbin/sysctl"
cmd = exec.Command("which", sbinPath)
_, _, _, err = myTarget.RunCommand(cmd, 0)
_, _, _, err = myTarget.RunCommand(cmd, 0, true)
if err == nil {
// found it
path = sbinPath
Expand Down
6 changes: 3 additions & 3 deletions cmd/metrics/process.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func GetCgroups(myTarget target.Target, cids []string, localTempDir string) (cgr
func GetHotProcesses(myTarget target.Target, maxProcesses int, filter string) (processes []Process, err error) {
// run ps to get list of processes sorted by cpu utilization (descending)
cmd := exec.Command("ps", "-a", "-x", "-h", "-o", "pid,ppid,comm,cmd", "--sort=-%cpu")
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0)
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true)
if err != nil {
err = fmt.Errorf("failed to get hot processes: %s, %d, %v", stderr, exitcode, err)
return
Expand Down Expand Up @@ -173,7 +173,7 @@ done | sort -nr | head -n %d

func processExists(myTarget target.Target, pid string) (exists bool) {
cmd := exec.Command("ps", "-p", pid)
_, _, _, err := myTarget.RunCommand(cmd, 0)
_, _, _, err := myTarget.RunCommand(cmd, 0, true)
if err != nil {
exists = false
return
Expand All @@ -184,7 +184,7 @@ func processExists(myTarget target.Target, pid string) (exists bool) {

func getProcess(myTarget target.Target, pid string) (process Process, err error) {
cmd := exec.Command("ps", "-q", pid, "h", "-o", "pid,ppid,comm,cmd", "ww")
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0)
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, true)
if err != nil {
err = fmt.Errorf("failed to get process: %s, %d, %v", stderr, exitcode, err)
return
Expand Down
6 changes: 3 additions & 3 deletions internal/script/script.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func RunScripts(myTarget target.Target, scripts []ScriptDefinition, ignoreScript
} else {
cmd = exec.Command("bash", path.Join(myTarget.GetTempDirectory(), masterScriptName))
}
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0)
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, false) // don't reuse ssh connection on long-running commands, makes it difficult to kill the command
if err != nil {
slog.Error("error running master script on target", slog.String("stdout", stdout), slog.String("stderr", stderr), slog.Int("exitcode", exitcode), slog.String("error", err.Error()))
return nil, err
Expand All @@ -183,7 +183,7 @@ func RunScripts(myTarget target.Target, scripts []ScriptDefinition, ignoreScript
// run sequential scripts
for _, script := range sequentialScripts {
cmd := prepareCommand(script, myTarget.GetTempDirectory())
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0)
stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, false)
if err != nil {
slog.Error("error running script on target", slog.String("script", script.Script), slog.String("stdout", stdout), slog.String("stderr", stderr), slog.Int("exitcode", exitcode), slog.String("error", err.Error()))
}
Expand Down Expand Up @@ -241,7 +241,7 @@ func RunScriptAsync(myTarget target.Target, script ScriptDefinition, localTempDi
}()
}
cmd := prepareCommand(script, myTarget.GetTempDirectory())
err = myTarget.RunCommandAsync(cmd, stdoutChannel, stderrChannel, exitcodeChannel, 0, cmdChannel)
err = myTarget.RunCommandAsync(cmd, 0, false, stdoutChannel, stderrChannel, exitcodeChannel, cmdChannel)
errorChannel <- err
}

Expand Down
68 changes: 40 additions & 28 deletions internal/target/target.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,24 @@ type Target interface {
GetUserPath() (path string, err error)

// RunCommand runs the specified command on the target.
// Arguments:
// - cmd: the command to run
// - timeout: the maximum time allowed for the command to run (zero means no timeout)
// - reuseSSHConnection: whether to reuse the SSH connection for the command (only relevant for RemoteTarget)
// It returns the standard output, standard error, exit code, and any error that occurred.
RunCommand(cmd *exec.Cmd, timeout int) (stdout string, stderr string, exitCode int, err error)
RunCommand(cmd *exec.Cmd, timeout int, reuseSSHConnection bool) (stdout string, stderr string, exitCode int, err error)

// RunCommandAsync runs the specified command on the target in an asynchronous manner.
// Arguments:
// - cmd: the command to run
// - timeout: the maximum time allowed for the command to run (zero means no timeout)
// - reuseSSHConnection: whether to reuse the SSH connection for the command (only relevant for RemoteTarget)
// - stdoutChannel: a channel to send the standard output of the command
// - stderrChannel: a channel to send the standard error of the command
// - exitcodeChannel: a channel to send the exit code of the command
// - cmdChannel: a channel to send the command that was run
// It returns any error that occurred.
RunCommandAsync(cmd *exec.Cmd, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, timeout int, cmdChannel chan *exec.Cmd) error
RunCommandAsync(cmd *exec.Cmd, timeout int, reuseSSHConnection bool, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, cmdChannel chan *exec.Cmd) error

// PushFile transfers a file from the local system to the target.
// It returns any error that occurred.
Expand Down Expand Up @@ -171,16 +183,16 @@ func (t *RemoteTarget) SetSshPass(sshPass string) {

// RunCommand executes the given command with a timeout and returns the standard output,
// standard error, exit code, and any error that occurred.
func (t *LocalTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, stderr string, exitCode int, err error) {
func (t *LocalTarget) RunCommand(cmd *exec.Cmd, timeout int, argNotUsed bool) (stdout string, stderr string, exitCode int, err error) {
input := ""
if t.sudo != "" && len(cmd.Args) > 2 && cmd.Args[0] == "sudo" && strings.HasPrefix(cmd.Args[1], "-") && strings.Contains(cmd.Args[1], "S") { // 'sudo -S' gets password from stdin
input = t.sudo + "\n"
}
return runLocalCommandWithInputWithTimeout(cmd, input, timeout)
}

func (t *RemoteTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, stderr string, exitCode int, err error) {
localCommand := t.prepareLocalCommand(cmd, false)
func (t *RemoteTarget) RunCommand(cmd *exec.Cmd, timeout int, reuseSSHConnection bool) (stdout string, stderr string, exitCode int, err error) {
localCommand := t.prepareLocalCommand(cmd, reuseSSHConnection)
return runLocalCommandWithInputWithTimeout(localCommand, "", timeout)
}

Expand All @@ -190,15 +202,15 @@ func (t *RemoteTarget) RunCommand(cmd *exec.Cmd, timeout int) (stdout string, st
// and the exit code is sent to the exitcodeChannel.
// The timeout parameter specifies the maximum time allowed for the command to run.
// Returns an error if there was a problem running the command.
func (t *LocalTarget) RunCommandAsync(cmd *exec.Cmd, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, timeout int, cmdChannel chan *exec.Cmd) (err error) {
func (t *LocalTarget) RunCommandAsync(cmd *exec.Cmd, timeout int, argNotUsed bool, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, cmdChannel chan *exec.Cmd) (err error) {
localCommand := cmd
cmdChannel <- localCommand
err = runLocalCommandWithInputWithTimeoutAsync(localCommand, stdoutChannel, stderrChannel, exitcodeChannel, "", timeout)
return
}

func (t *RemoteTarget) RunCommandAsync(cmd *exec.Cmd, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, timeout int, cmdChannel chan *exec.Cmd) (err error) {
localCommand := t.prepareLocalCommand(cmd, true)
func (t *RemoteTarget) RunCommandAsync(cmd *exec.Cmd, timeout int, reuseSSHConnection bool, stdoutChannel chan string, stderrChannel chan string, exitcodeChannel chan int, cmdChannel chan *exec.Cmd) (err error) {
localCommand := t.prepareLocalCommand(cmd, reuseSSHConnection)
cmdChannel <- localCommand
err = runLocalCommandWithInputWithTimeoutAsync(localCommand, stdoutChannel, stderrChannel, exitcodeChannel, "", timeout)
return
Expand Down Expand Up @@ -266,7 +278,7 @@ func (t *RemoteTarget) CreateTempDirectory(rootDir string) (tempDir string, err
root = fmt.Sprintf("--tmpdir=%s", rootDir)
}
cmd := exec.Command("mktemp", "-d", "-t", root, "perfspect.tmp.XXXXXXXXXX", "|", "xargs", "realpath")
tempDir, _, _, err = t.RunCommand(cmd, 0)
tempDir, _, _, err = t.RunCommand(cmd, 0, true)
tempDir = strings.TrimSpace(tempDir)
t.tempDir = tempDir
return
Expand Down Expand Up @@ -331,7 +343,7 @@ func (t *LocalTarget) CreateDirectory(baseDir string, targetDir string) (dir str
func (t *RemoteTarget) CreateDirectory(baseDir string, targetDir string) (dir string, err error) {
dir = filepath.Join(baseDir, targetDir)
cmd := exec.Command("mkdir", dir)
_, _, _, err = t.RunCommand(cmd, 0)
_, _, _, err = t.RunCommand(cmd, 0, true)
return
}

Expand All @@ -348,7 +360,7 @@ func (t *LocalTarget) RemoveDirectory(targetDir string) (err error) {
func (t *RemoteTarget) RemoveDirectory(targetDir string) (err error) {
if targetDir != "" {
cmd := exec.Command("rm", "-rf", targetDir)
_, _, _, err = t.RunCommand(cmd, 0)
_, _, _, err = t.RunCommand(cmd, 0, true)
}
return
}
Expand All @@ -360,7 +372,7 @@ func (t *LocalTarget) CanConnect() bool {

func (t *RemoteTarget) CanConnect() bool {
cmd := exec.Command("exit", "0")
_, _, _, err := t.RunCommand(cmd, 5)
_, _, _, err := t.RunCommand(cmd, 5, true)
return err == nil
}

Expand Down Expand Up @@ -388,14 +400,14 @@ func (t *LocalTarget) CanElevatePrivileges() bool {
slog.Error("error writing sudo password", slog.String("error", err.Error()))
}
}()
_, _, _, err := t.RunCommand(cmd, 0)
_, _, _, err := t.RunCommand(cmd, 0, true)
if err == nil {
t.canElevate = 1
return true // sudo password works
}
}
cmd := exec.Command("sudo", "-kS", "ls")
_, _, _, err := t.RunCommand(cmd, 0)
_, _, _, err := t.RunCommand(cmd, 0, true)
if err == nil { // true - passwordless sudo works
t.canElevate = 1
return true
Expand All @@ -415,7 +427,7 @@ func (t *RemoteTarget) CanElevatePrivileges() bool {
return true
}
cmd := exec.Command("sudo", "-kS", "ls")
_, _, _, err := t.RunCommand(cmd, 0)
_, _, _, err := t.RunCommand(cmd, 0, true)
if err == nil { // true - passwordless sudo works
t.canElevate = 1
return true
Expand Down Expand Up @@ -492,7 +504,7 @@ func (t *LocalTarget) GetUserPath() (string, error) {
func (t *RemoteTarget) GetUserPath() (string, error) {
if t.userPath == "" {
cmd := exec.Command("echo", "$PATH")
stdout, _, _, err := t.RunCommand(cmd, 0)
stdout, _, _, err := t.RunCommand(cmd, 0, true)
if err != nil {
return "", err
}
Expand Down Expand Up @@ -594,7 +606,7 @@ func runLocalCommandWithInputWithTimeoutAsync(cmd *exec.Cmd, stdoutChannel chan
return nil
}

func (t *RemoteTarget) prepareSSHFlags(scp bool, async bool, prompt bool) (flags []string) {
func (t *RemoteTarget) prepareSSHFlags(scp bool, useControlMaster bool, prompt bool) (flags []string) {
flags = []string{
"-2",
"-o",
Expand All @@ -621,7 +633,7 @@ func (t *RemoteTarget) prepareSSHFlags(scp bool, async bool, prompt bool) (flags
flags = append(flags, promptFlags...)
}
// when using a control master, a long-running remote program doesn't get terminated when the local ssh client is terminated
if !async {
if useControlMaster {
controlPathFlags := []string{
"-o",
"ControlPath=" + filepath.Join(os.TempDir(), `control-%h-%p-%r`),
Expand Down Expand Up @@ -654,10 +666,10 @@ func (t *RemoteTarget) prepareSSHFlags(scp bool, async bool, prompt bool) (flags
return
}

func (t *RemoteTarget) prepareSSHCommand(command []string, async bool, prompt bool) []string {
func (t *RemoteTarget) prepareSSHCommand(command []string, useControlMaster bool, prompt bool) []string {
var cmd []string
cmd = append(cmd, "ssh")
cmd = append(cmd, t.prepareSSHFlags(false, async, prompt)...)
cmd = append(cmd, t.prepareSSHFlags(false, useControlMaster, prompt)...)
if t.user != "" {
cmd = append(cmd, t.user+"@"+t.host)
} else {
Expand All @@ -671,7 +683,7 @@ func (t *RemoteTarget) prepareSSHCommand(command []string, async bool, prompt bo
func (t *RemoteTarget) prepareSCPCommand(src string, dstDir string, push bool) []string {
var cmd []string
cmd = append(cmd, "scp")
cmd = append(cmd, t.prepareSSHFlags(true, false, false)...)
cmd = append(cmd, t.prepareSSHFlags(true, true, false)...)
if push {
fileInfo, err := os.Stat(src)
if err != nil {
Expand All @@ -698,11 +710,11 @@ func (t *RemoteTarget) prepareSCPCommand(src string, dstDir string, push bool) [
return cmd
}

func (t *RemoteTarget) prepareLocalCommand(cmd *exec.Cmd, async bool) *exec.Cmd {
func (t *RemoteTarget) prepareLocalCommand(cmd *exec.Cmd, useControlMaster bool) *exec.Cmd {
var name string
var args []string
usePass := t.key == "" && t.sshPass != ""
sshCommand := t.prepareSSHCommand(cmd.Args, async, usePass)
sshCommand := t.prepareSSHCommand(cmd.Args, useControlMaster, usePass)
if usePass {
name = t.sshpassPath
args = []string{"-e", "--"}
Expand Down Expand Up @@ -741,7 +753,7 @@ func (t *RemoteTarget) prepareAndRunSCPCommand(srcPath string, dstDir string, is

func getArchitecture(t Target) (arch string, err error) {
cmd := exec.Command("uname", "-m")
arch, _, _, err = t.RunCommand(cmd, 0)
arch, _, _, err = t.RunCommand(cmd, 0, true)
if err != nil {
return
}
Expand All @@ -751,7 +763,7 @@ func getArchitecture(t Target) (arch string, err error) {

func getFamily(t Target) (family string, err error) {
cmd := exec.Command("bash", "-c", "lscpu | grep -i \"^CPU family:\" | awk '{print $NF}'")
family, _, _, err = t.RunCommand(cmd, 0)
family, _, _, err = t.RunCommand(cmd, 0, true)
if err != nil {
return
}
Expand All @@ -761,7 +773,7 @@ func getFamily(t Target) (family string, err error) {

func getModel(t Target) (model string, err error) {
cmd := exec.Command("bash", "-c", "lscpu | grep -i model: | awk '{print $NF}'")
model, _, _, err = t.RunCommand(cmd, 0)
model, _, _, err = t.RunCommand(cmd, 0, true)
if err != nil {
return
}
Expand All @@ -776,7 +788,7 @@ func installLkms(t Target, lkms []string) (installedLkms []string, err error) {
}
for _, lkm := range lkms {
slog.Debug("attempting to install kernel module", slog.String("lkm", lkm))
_, _, _, err := t.RunCommand(exec.Command("modprobe", "--first-time", lkm), 10)
_, _, _, err := t.RunCommand(exec.Command("modprobe", "--first-time", lkm), 10, true)
if err != nil {
slog.Debug("kernel module already installed or problem installing", slog.String("lkm", lkm), slog.String("error", err.Error()))
continue
Expand All @@ -794,7 +806,7 @@ func uninstallLkms(t Target, lkms []string) (err error) {
}
for _, lkm := range lkms {
slog.Debug("attempting to uninstall kernel module", slog.String("lkm", lkm))
_, _, _, err := t.RunCommand(exec.Command("modprobe", "-r", lkm), 10)
_, _, _, err := t.RunCommand(exec.Command("modprobe", "-r", lkm), 10, true)
if err != nil {
slog.Error("error uninstalling kernel module", slog.String("lkm", lkm), slog.String("error", err.Error()))
continue
Expand Down

0 comments on commit de2ff60

Please sign in to comment.