Sequester error to individual hosts

client/controller.go

@@ -69,24 +69,61 @@ func (hosts *HostsController) setReadOnlyHost(hostlist config.HostList) {
 	hosts.ReadOnlyHosts = hostlist
 }
 
+func (hosts *HostsController) handleError(err error, metric string, host config.Host, client *Client) {
+	var errorContent string
+	if !strings.Contains(fmt.Sprintf("%s", err), "127") {
+		errorContent = fmt.Sprintf("Could not retrieve metric %s from driver %s with error %s", metric, host.Address, err)
+	} else {
+		errorContent = fmt.Sprintf("Command %s not found on driver %s", metric, host.Address)
+	}
+	log.Debug(errorContent)
+	//FIXME: what kind of errors do we especially want to reset driver for
+	if _, ok := err.(*driver.SSHConnectError); ok {
+		hosts.resetDriver(host)
+	}
+	message := &SendMessage{
+		Message: ErrorMessage{
+			Error: errorContent,
+			Host:  host.Address,
+			Name:  metric,
+		},
+		Error: true,
+	}
+	client.Send <- message
+}
+
 func (hosts *HostsController) sendMetric(host config.Host, client *Client) {
+	var (
+		err               error
+		data              []byte
+		initializedMetric inspector.Inspector
+		platformDetails   driver.SystemDetails
+	)
 	if hosts.getDriver(host.Address) == nil {
 		hosts.resetDriver(host)
 	}
 	for metric, custom := range hosts.Info.Metrics {
 		inspectorDriver := hosts.getDriver(host.Address)
-		initializedMetric, err := inspector.Init(metric, inspectorDriver, custom)
+		initializedMetric, err = inspector.Init(metric, inspectorDriver, custom)
 		if err != nil {
 			log.Error(err)
+			hosts.handleError(err, metric, host, client)
+			continue
 		}
-		data, err := initializedMetric.Execute()
+		platformDetails, err = (*inspectorDriver).GetDetails()
+		if err != nil {
+			log.Error(err)
+			hosts.handleError(err, metric, host, client)
+			continue
+		}
+		data, err = initializedMetric.Execute()
 		if err == nil {
 			var unmarsh interface{}
 			json.Unmarshal(data, &unmarsh)
 			message := &SendMessage{
 				Message: Message{
 					Host:     host.Address,
-					Platform: (*inspectorDriver).GetDetails().Name,
+					Platform: platformDetails.Name,
 					Name:     metric,
 					Data:     unmarsh,
 				},
@@ -96,27 +133,7 @@ func (hosts *HostsController) sendMetric(host config.Host, client *Client) {
 				client.Send <- message
 			}
 		} else {
-			// check for error 127 which means command was not found
-			var errorContent string
-			if !strings.Contains(fmt.Sprintf("%s", err), "127") {
-				errorContent = fmt.Sprintf("Could not retrieve metric %s from driver %s with error %s", metric, host.Address, err)
-			} else {
-				errorContent = fmt.Sprintf("Command %s not found on driver %s", metric, host.Address)
-			}
-			log.Debug(errorContent)
-			//FIXME: what kind of errors do we especially want to reset driver for
-			if _, ok := err.(*driver.SSHConnectError); ok {
-				hosts.resetDriver(host)
-			}
-			message := &SendMessage{
-				Message: ErrorMessage{
-					Error: errorContent,
-					Host:  host.Address,
-					Name:  metric,
-				},
-				Error: true,
-			}
-			client.Send <- message
+			hosts.handleError(err, metric, host, client)
 		}
 	}
 }

driver/driver.go

@@ -26,7 +26,7 @@ type Driver interface {
 	ReadFile(path string) (string, error)
 	RunCommand(command string) (string, error)
 	// shows the driver details, not sure if we should be showing OS name
-	GetDetails() SystemDetails
+	GetDetails() (SystemDetails, error)
 }
 
 func ToDriver(conn config.Connection) Driver {

driver/local.go

@@ -36,7 +36,10 @@ func (d *Local) RunCommand(command string) (string, error) {
 	var cmd *exec.Cmd
 	log.Debugf("Running command `%s` ", command)
 	if d.Info == nil {
-		d.GetDetails()
+		_, err := d.GetDetails()
+		if err != nil {
+			return ``, err
+		}
 	}
 	if d.Info.IsLinux || d.Info.IsDarwin {
 		cmd = exec.Command("bash", "-c", command)
@@ -58,7 +61,7 @@ func (d *Local) RunCommand(command string) (string, error) {
 	return string(out), nil
 }
 
-func (d *Local) GetDetails() SystemDetails {
+func (d *Local) GetDetails() (SystemDetails, error) {
 	if d.Info == nil {
 		details := &SystemDetails{}
 		details.Name = strings.Title(runtime.GOOS)
@@ -73,5 +76,5 @@ func (d *Local) GetDetails() SystemDetails {
 		details.Extra = runtime.GOARCH
 		d.Info = details
 	}
-	return *d.Info
+	return *d.Info, nil
 }

driver/local_unix_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package driver
@@ -17,8 +18,8 @@ func TestUnixLocalRunCommand(t *testing.T) {
 
 func TestUnixLocalSystemDetails(t *testing.T) {
 	d := Local{}
-	details := d.GetDetails()
-	if !(details.IsLinux || details.IsDarwin) {
+	details, err := d.GetDetails()
+	if err != nil || !(details.IsLinux || details.IsDarwin) {
 		t.Errorf("Expected Darwin or Linux on unix test, got %s", details.Name)
 	}
 }

driver/local_windows_test.go

@@ -15,8 +15,8 @@ func TestWindowsRunCommand(t *testing.T) {
 
 func TestWindowsLocalSystemDetails(t *testing.T) {
 	d := Local{}
-	details := d.GetDetails()
-	if !details.IsWindows {
+	details, err := d.GetDetails()
+	if err != nil || !details.IsWindows {
 		t.Errorf("Expected windows got %s", details.Name)
 	}
 }

driver/ssh.go

@@ -129,7 +129,7 @@ func (d *SSH) RunCommand(command string) (string, error) {
 	return string(out), nil
 }
 
-func (d *SSH) GetDetails() SystemDetails {
+func (d *SSH) GetDetails() (SystemDetails, error) {
 	if d.Info == nil {
 		// TODO: Check for goph specific errors
 		// within RunCommand and only return errors that are not
@@ -145,7 +145,8 @@ func (d *SSH) GetDetails() SystemDetails {
 				}
 			} else {
 				//FIXME: Fix issue with establishing connection on SSH
-				panic(fmt.Sprintf("Could not find platform details for %s: %s", d.Host, err))
+				log.Errorf("Could not find platform details for %s: %s", d.Host, err)
+				return SystemDetails{}, err
 			}
 		}
 		details := &SystemDetails{}
@@ -160,5 +161,5 @@ func (d *SSH) GetDetails() SystemDetails {
 		}
 		d.Info = details
 	}
-	return *d.Info
+	return *d.Info, nil
 }

driver/ssh_test.go

@@ -58,8 +58,8 @@ func TestSSHSystemDetails(t *testing.T) {
 		return
 	}
 	d := NewSSHForTest()
-	details := d.GetDetails()
-	if !details.IsLinux {
+	details, err := d.GetDetails()
+	if err != nil || !details.IsLinux {
 		t.Errorf("Expected linux server for ssh test got %#v", details)
 	}
 }

driver/web.go

@@ -62,7 +62,7 @@ func (d *Web) RunCommand(command string) (string, error) {
 	return ``, errors.New("Cannot read file on web driver")
 }
 
-func (d *Web) GetDetails() SystemDetails {
+func (d *Web) GetDetails() (SystemDetails, error) {
 	if d.Info == nil {
 		details := &SystemDetails{
 			Name:  "web",
@@ -71,5 +71,5 @@ func (d *Web) GetDetails() SystemDetails {
 		}
 		d.Info = details
 	}
-	return *d.Info
+	return *d.Info, nil
 }

driver/web_test.go

@@ -27,8 +27,8 @@ func TestWebRunCommand(t *testing.T) {
 
 func TestWebSystemDetails(t *testing.T) {
 	d := NewWebForTest()
-	details := d.GetDetails()
-	if !details.IsWeb {
+	details, err := d.GetDetails()
+	if err != nil || !details.IsWeb {
 		t.Errorf("Expected web driver for web test got %s", details.Name)
 	}
 }

inspector/custom.go

@@ -36,7 +36,7 @@ func (i Custom) createMetric(output string) CustomMetrics {
 }
 
 func (i *Custom) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if details.IsWeb {
 		panic(fmt.Sprintf("Cannot use Custom(%s) on web", i.Command))
 	}
@@ -59,7 +59,10 @@ func (i *Custom) Execute() ([]byte, error) {
 // NewCustom : Initialize a new Custom instance
 func NewCustom(driver *driver.Driver, custom ...string) (Inspector, error) {
 	var customInspector Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if details.IsWeb {
 		return nil, errors.New(fmt.Sprintf("Cannot use Custom(%s) on web", custom))
 	}

inspector/disk.go

@@ -212,7 +212,10 @@ func (i *DFWin) Execute() ([]byte, error) {
 // NewDF : Initialize a new DF instance
 func NewDF(driver *driver.Driver, _ ...string) (Inspector, error) {
 	var df Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if !(details.IsLinux || details.IsDarwin || details.IsWindows) {
 		return nil, errors.New("Cannot use 'df' command on drivers outside (linux, darwin, windows)")
 	}

inspector/docker_stats.go

@@ -40,7 +40,7 @@ redis2     0.07%     2.746 MB / 64 MB      4.29%     1.266 KB / 648 B    12.4 MB
 func (i *DockerStats) Parse(output string) {
 	var values []DockerStatsMetrics
 	var splitChars string
-	details := (*i.Driver).GetDetails()
+	details, _ := (*i.Driver).GetDetails()
 	if details.IsWindows {
 		splitChars = "\r\n"
 	} else {
@@ -121,7 +121,10 @@ func (i *DockerStats) Execute() ([]byte, error) {
 // NewDockerStats : Initialize a new DockerStats instance
 func NewDockerStats(driver *driver.Driver, _ ...string) (Inspector, error) {
 	var dockerstats Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if !(details.IsLinux || details.IsDarwin || details.IsWindows) {
 		return nil, errors.New("Cannot use LoadAvgDarwin on drivers outside (linux, darwin, windows)")
 	}

inspector/loadavg.go

@@ -58,7 +58,7 @@ func loadavgParseOutput(output string) *LoadAvgMetrics {
 }
 
 func (i *LoadAvgDarwin) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsDarwin {
 		panic("Cannot use LoadAvgDarwin on drivers outside (darwin)")
 	}
@@ -97,7 +97,7 @@ func (i *LoadAvgLinux) Parse(output string) {
 }
 
 func (i *LoadAvgLinux) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsLinux {
 		panic("Cannot use LoadAvg on drivers outside (linux)")
 	}
@@ -130,7 +130,7 @@ func (i *LoadAvgWin) Parse(output string) {
 }
 
 func (i *LoadAvgWin) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsWindows {
 		panic("Cannot use LoadAvgWin on drivers outside (windows)")
 	}
@@ -154,7 +154,10 @@ func (i *LoadAvgWin) Execute() ([]byte, error) {
 // NewLoadAvg : Initialize a new LoadAvg instance
 func NewLoadAvg(driver *driver.Driver, _ ...string) (Inspector, error) {
 	var loadavg Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if !(details.IsLinux || details.IsDarwin || details.IsWindows) {
 		return nil, errors.New("Cannot use LoadAvg on drivers outside (linux, darwin)")
 	}

inspector/meminfo.go

@@ -118,7 +118,7 @@ func (i *MemInfoLinux) Parse(output string) {
 }
 
 func (i *MemInfoLinux) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsLinux {
 		panic("Cannot use MeminfoLinux outside (linux)")
 	}
@@ -186,7 +186,7 @@ func (i *MemInfoDarwin) Parse(output string) {
 }
 
 func (i *MemInfoDarwin) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsDarwin {
 		panic("Cannot use MeminfoDarwin outside (darwin)")
 	}
@@ -266,7 +266,7 @@ func (i *MemInfoWin) Parse(output string) {
 }
 
 func (i *MemInfoWin) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsWindows {
 		panic("Cannot use MeminfoWin outside (windows)")
 	}
@@ -299,7 +299,10 @@ func (i *MemInfoWin) Execute() ([]byte, error) {
 // NewMemInfoLinux : Initialize a new MemInfoLinux instance
 func NewMemInfo(driver *driver.Driver, _ ...string) (Inspector, error) {
 	var meminfo Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if !(details.IsLinux || details.IsDarwin || details.IsWindows) {
 		return nil, errors.New("Cannot use MemInfo on drivers outside (linux, darwin, windows)")
 	}

inspector/process.go

@@ -50,7 +50,7 @@ type ProcessWin struct {
 }
 
 func (i *Process) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !(details.IsLinux || details.IsDarwin) {
 		panic("Cannot use Process on drivers outside (linux, darwin)")
 	}
@@ -197,7 +197,7 @@ func (i *ProcessWin) createMetric(columns []string, pid int) ProcessMetricsWin {
 }
 
 func (i *ProcessWin) SetDriver(driver *driver.Driver) {
-	details := (*driver).GetDetails()
+	details, _ := (*driver).GetDetails()
 	if !details.IsWindows {
 		panic("Cannot use ProcessWin on drivers outside (windows)")
 	}
@@ -220,7 +220,10 @@ func (i *ProcessWin) Execute() ([]byte, error) {
 // NewProcess : Initialize a new Process instance
 func NewProcess(driver *driver.Driver, _ ...string) (Inspector, error) {
 	var process Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if !(details.IsLinux || details.IsDarwin || details.IsWindows) {
 		return nil, errors.New("Cannot use Process on drivers outside (linux, darwin, windows)")
 	}

inspector/rt.go

@@ -55,7 +55,10 @@ func (i *ResponseTime) Execute() ([]byte, error) {
 // NewResponseTime : Initialize a new ResponseTime instance
 func NewResponseTime(driver *driver.Driver, _ ...string) (Inspector, error) {
 	var responsetime Inspector
-	details := (*driver).GetDetails()
+	details, err := (*driver).GetDetails()
+	if err != nil {
+		return nil, err
+	}
 	if !(details.IsWeb) {
 		return nil, errors.New("Cannot use response time outside driver (web)")
 	}