Merge pull request #1652 from bosun-monitor/maxMemConf

cmd/scollector: Make MaxMem kill switch configurable

cmd/scollector/conf/conf.go

@@ -25,6 +25,11 @@ type Conf struct {
 	BatchSize int
 	// MaxQueueLen is the number of metrics keept internally.
 	MaxQueueLen int
+	// MaxMem is the maximum number of megabytes that can be allocated
+	// before scollector panics (shuts down). Default of 500 MB. This
+	// is a saftey mechanism to protect the host from the monitoring
+	// agent
+	MaxMem uint64
 	// Filter filters collectors matching these terms.
 	Filter []string
 	// PProf is an IP:Port binding to be used for debugging with pprof package.

cmd/scollector/main.go

@@ -240,14 +240,17 @@ func main() {
 		}
 		collect.MaxQueueLen = conf.MaxQueueLen
 	}
-
+	maxMemMegaBytes := uint64(500)
+	if conf.MaxMem != 0 {
+		maxMemMegaBytes = conf.MaxMem
+	}
 	go func() {
-		const maxMem = 500 * 1024 * 1024 // 500MB
+		maxMemBytes := maxMemMegaBytes * 1024 * 1024
 		var m runtime.MemStats
-		for range time.Tick(time.Minute) {
+		for range time.Tick(time.Second * 30) {
 			runtime.ReadMemStats(&m)
-			if m.Alloc > maxMem {
-				panic("memory max reached")
+			if m.Alloc > maxMemBytes {
+				panic(fmt.Sprintf("memory max reached: (current: %v bytes, max: %v bytes)", m.Alloc, maxMemBytes))
 			}
 		}
 	}()