support buffer limit option in FireLens

aws · Jul 21, 2021 · 7a79689 · 7a79689
1 parent 997c975
commit 7a79689
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 4 deletions.
diff --git a/agent/api/task/task.go b/agent/api/task/task.go
@@ -117,6 +117,8 @@ const (
 	firelensSocketBindFormat = "%s/data/firelens/%s/socket/:/var/run/"
 	// firelensDriverName is the log driver name for containers that want to use the firelens container to send logs.
 	firelensDriverName = "awsfirelens"
+	// FirelensLogDriverBufferLimitOption is the option for customers who want to specify the buffer limit size in FireLens.
+	FirelensLogDriverBufferLimitOption = "log-driver-buffer-limit"
 
 	// firelensConfigVarFmt specifies the format for firelens config variable name. The first placeholder
 	// is option name. The second placeholder is the index of the container in the task's container list, appended
@@ -1069,6 +1071,9 @@ func (task *Task) collectFirelensLogOptions(containerToLogOptions map[string]map
 				containerToLogOptions[container.Name] = make(map[string]string)
 			}
 			for k, v := range hostConfig.LogConfig.Config {
+				if k == FirelensLogDriverBufferLimitOption {
+					continue
+				}
 				containerToLogOptions[container.Name][k] = v
 			}
 		}

diff --git a/agent/api/task/task_linux_test.go b/agent/api/task/task_linux_test.go
@@ -1148,8 +1148,9 @@ func getFirelensTask(t *testing.T) *Task {
 		LogConfig: dockercontainer.LogConfig{
 			Type: firelensDriverName,
 			Config: map[string]string{
-				"key1": "value1",
-				"key2": "value2",
+				"key1":                    "value1",
+				"key2":                    "value2",
+				"log-driver-buffer-limit": "10000",
 			},
 		},
 	}

diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go
@@ -89,6 +89,7 @@ const (
 	dataLogDriverPath           = "/data/firelens/"
 	logDriverAsyncConnect       = "fluentd-async-connect"
 	logDriverSubSecondPrecision = "fluentd-sub-second-precision"
+	logDriverBufferLimit        = "fluentd-buffer-limit"
 	dataLogDriverSocketPath     = "/socket/fluent.sock"
 	socketPathPrefix            = "unix://"
 
@@ -1279,12 +1280,16 @@ func getFirelensLogConfig(task *apitask.Task, container *apicontainer.Container,
 	tag := fmt.Sprintf(fluentTagDockerFormat, container.Name, taskID)
 	fluentd := socketPathPrefix + filepath.Join(cfg.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath)
 	logConfig := hostConfig.LogConfig
+	bufferLimit, bufferLimitExists := logConfig.Config[apitask.FirelensLogDriverBufferLimitOption]
 	logConfig.Type = logDriverTypeFluentd
 	logConfig.Config = make(map[string]string)
 	logConfig.Config[logDriverTag] = tag
 	logConfig.Config[logDriverFluentdAddress] = fluentd
 	logConfig.Config[logDriverAsyncConnect] = strconv.FormatBool(true)
 	logConfig.Config[logDriverSubSecondPrecision] = strconv.FormatBool(true)
+	if bufferLimitExists {
+		logConfig.Config[logDriverBufferLimit] = bufferLimit
+	}
 	seelog.Debugf("Applying firelens log config for container %s: %v", container.Name, logConfig)
 	return logConfig
 }

diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go
@@ -2670,8 +2670,9 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			LogConfig: dockercontainer.LogConfig{
 				Type: logDriverType,
 				Config: map[string]string{
-					"key1": "value1",
-					"key2": "value2",
+					"key1":                    "value1",
+					"key2":                    "value2",
+					"log-driver-buffer-limit": "10000",
 				},
 			},
 		}
@@ -2766,6 +2767,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 		expectedLogConfigFluentAddress string
 		expectedFluentdAsyncConnect    string
 		expectedSubSecondPrecision     string
+		expectedBufferLimit            string
 		expectedIPAddress              string
 		expectedPort                   string
 	}{
@@ -2776,6 +2778,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			expectedLogConfigTag:           taskName + "-firelens-" + taskID,
 			expectedFluentdAsyncConnect:    strconv.FormatBool(true),
 			expectedSubSecondPrecision:     strconv.FormatBool(true),
+			expectedBufferLimit:            "10000",
 			expectedLogConfigFluentAddress: socketPathPrefix + filepath.Join(defaultConfig.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath),
 			expectedIPAddress:              envVarBridgeMode,
 			expectedPort:                   envVarPort,
@@ -2787,6 +2790,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			expectedLogConfigTag:           taskName + "-firelens-" + taskID,
 			expectedFluentdAsyncConnect:    strconv.FormatBool(true),
 			expectedSubSecondPrecision:     strconv.FormatBool(true),
+			expectedBufferLimit:            "10000",
 			expectedLogConfigFluentAddress: socketPathPrefix + filepath.Join(defaultConfig.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath),
 			expectedIPAddress:              envVarBridgeMode,
 			expectedPort:                   envVarPort,
@@ -2798,6 +2802,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			expectedLogConfigTag:           taskName + "-firelens-" + taskID,
 			expectedFluentdAsyncConnect:    strconv.FormatBool(true),
 			expectedSubSecondPrecision:     strconv.FormatBool(true),
+			expectedBufferLimit:            "",
 			expectedLogConfigFluentAddress: socketPathPrefix + filepath.Join(defaultConfig.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath),
 			expectedIPAddress:              envVarAWSVPCMode,
 			expectedPort:                   envVarPort,
@@ -2823,6 +2828,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 					assert.Equal(t, tc.expectedLogConfigFluentAddress, hostConfig.LogConfig.Config["fluentd-address"])
 					assert.Equal(t, tc.expectedFluentdAsyncConnect, hostConfig.LogConfig.Config["fluentd-async-connect"])
 					assert.Equal(t, tc.expectedSubSecondPrecision, hostConfig.LogConfig.Config["fluentd-sub-second-precision"])
+					assert.Equal(t, tc.expectedBufferLimit, hostConfig.LogConfig.Config["fluentd-buffer-limit"])
 					assert.Contains(t, config.Env, tc.expectedIPAddress)
 					assert.Contains(t, config.Env, tc.expectedPort)
 				})