input/dockerstats: use log_mode instead of zero_hierarchical_memory_l…

…imit

Signed-off-by: tsaikd <tsaikd@gmail.com>

config/logevent/logevent.go

@@ -1,12 +1,13 @@
 package logevent
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"regexp"
 	"strings"
 	"time"
+
+	"github.com/tsaikd/KDGoLib/jsonex"
 )
 
 type LogEvent struct {
@@ -52,12 +53,12 @@ func (t LogEvent) getJSONMap() map[string]interface{} {
 
 func (t LogEvent) MarshalJSON() (data []byte, err error) {
 	event := t.getJSONMap()
-	return json.Marshal(event)
+	return jsonex.Marshal(event)
 }
 
 func (t LogEvent) MarshalIndent() (data []byte, err error) {
 	event := t.getJSONMap()
-	return json.MarshalIndent(event, "", "\t")
+	return jsonex.MarshalIndent(event, "", "\t")
 }
 
 func (t LogEvent) Get(field string) (v interface{}) {

input/dockerstats/README.md

@@ -24,8 +24,8 @@ gogstash input dockerstats
 			// (optional), in seconds, docker connection retry interval, default: 10
 			"connection_retry_interval": 10
 
-			// (optional), zero hierarchical_memory_limit in docker stats, sometimes it will cause elastic exception 'JsonParseException Numeric value out of range of long', default: false
-			"zero_hierarchical_memory_limit": false
+			// (optional), filter the output by mode, available value: "full" | "simple", "simple" mode will remove some messages to make the log smaller, default: "full"
+			"log_mode": "full"
 		}
 	]
 }

input/dockerstats/inputdockerstats.go

@@ -23,13 +23,13 @@ type InputConfig struct {
 	StatInterval            int      `json:"stat_interval"`
 	ConnectionRetryInterval int      `json:"connection_retry_interval,omitempty"`
 
-	ZeroHierarchicalMemoryLimit bool `json:"zero_hierarchical_memory_limit,omitempty"`
+	LogMode Mode `json:"log_mode,omitempty"`
 
-	sincemap map[string]*time.Time `json:"-"`
-	includes []*regexp.Regexp      `json:"-"`
-	excludes []*regexp.Regexp      `json:"-"`
-	hostname string                `json:"-"`
-	client   *docker.Client        `json:"-"`
+	sincemap map[string]*time.Time
+	includes []*regexp.Regexp
+	excludes []*regexp.Regexp
+	hostname string
+	client   *docker.Client
 }
 
 func DefaultInputConfig() InputConfig {
@@ -42,6 +42,7 @@ func DefaultInputConfig() InputConfig {
 		DockerURL:               "unix:///var/run/docker.sock",
 		StatInterval:            15,
 		ConnectionRetryInterval: 10,
+		LogMode:                 ModeFull,
 
 		sincemap: map[string]*time.Time{},
 	}
@@ -126,8 +127,6 @@ func (t *InputConfig) start(logger *logrus.Logger, inchan config.InChan) (err er
 			}
 		}
 	}
-
-	return
 }
 
 func (t *InputConfig) isValidContainer(names []string) bool {
@@ -145,7 +144,6 @@ func (t *InputConfig) isValidContainer(names []string) bool {
 	}
 	if len(t.includes) > 0 {
 		return false
-	} else {
-		return true
 	}
+	return true
 }

input/dockerstats/logloop.go

@@ -1,6 +1,7 @@
 package inputdockerstats
 
 import (
+	"reflect"
 	"strings"
 	"time"
 
@@ -51,9 +52,7 @@ func (t *InputConfig) containerLogLoop(container interface{}, since *time.Time,
 						continue
 					}
 
-					if t.ZeroHierarchicalMemoryLimit {
-						stats.MemoryStats.Stats.HierarchicalMemoryLimit = 0
-					}
+					filterStatsByMode(stats, t.LogMode)
 
 					event := logevent.LogEvent{
 						Timestamp: time.Now(),
@@ -85,3 +84,32 @@ func (t *InputConfig) containerLogLoop(container interface{}, since *time.Time,
 
 	return
 }
+
+func filterStatsByMode(stats *docker.Stats, mode Mode) {
+	switch mode {
+	case ModeSimple:
+		clearNetworkStats(&stats.Network)
+		for name, network := range stats.Networks {
+			clearNetworkStats(&network)
+			stats.Networks[name] = network
+		}
+		clear(&stats.MemoryStats.Stats)
+		clear(&stats.BlkioStats)
+		clear(&stats.CPUStats.CPUUsage.PercpuUsage)
+		clear(&stats.CPUStats.CPUUsage.UsageInKernelmode)
+		clear(&stats.CPUStats.CPUUsage.UsageInUsermode)
+		clear(&stats.CPUStats.SystemCPUUsage)
+	}
+}
+
+func clearNetworkStats(network *docker.NetworkStats) {
+	*network = docker.NetworkStats{
+		RxBytes: network.RxBytes,
+		TxBytes: network.TxBytes,
+	}
+}
+
+func clear(v interface{}) {
+	p := reflect.ValueOf(v).Elem()
+	p.Set(reflect.Zero(p.Type()))
+}

input/dockerstats/mode.go

@@ -0,0 +1,45 @@
+package inputdockerstats
+
+import "github.com/tsaikd/KDGoLib/enumutil"
+
+// Mode enum for docker stats log mode
+type Mode int8
+
+// List all valid enum of Mode
+const (
+	ModeFull Mode = 1 + iota
+	ModeSimple
+)
+
+var factoryMode = enumutil.NewEnumFactory().
+	Add(ModeFull, "full").
+	Add(ModeSimple, "simple").
+	Build()
+
+func (t Mode) String() string {
+	return factoryMode.String(t)
+}
+
+// MarshalJSON return jsonfy []byte of enum
+func (t Mode) MarshalJSON() ([]byte, error) {
+	return factoryMode.MarshalJSON(t)
+}
+
+// UnmarshalJSON decode json data to enum
+func (t *Mode) UnmarshalJSON(b []byte) (err error) {
+	return factoryMode.UnmarshalJSON(t, b)
+}
+
+// IsMode check string is valid enum
+func IsMode(s string) bool {
+	return factoryMode.IsEnumString(s)
+}
+
+// ParseMode string to enum
+func ParseMode(s string) Mode {
+	enum, err := factoryMode.ParseString(s)
+	if err != nil {
+		return 0
+	}
+	return enum.(Mode)
+}