From eba442b29e429f2900ae6adacc818cd1e7d3a662 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Fri, 15 Mar 2024 10:34:17 -0700 Subject: [PATCH 1/5] init: refactor env var prefixes to const Signed-off-by: Wesley Pettit --- init/fluent_bit_init_process.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/init/fluent_bit_init_process.go b/init/fluent_bit_init_process.go index 1e651da4b..c78418e74 100644 --- a/init/fluent_bit_init_process.go +++ b/init/fluent_bit_init_process.go @@ -19,6 +19,12 @@ import ( "github.com/sirupsen/logrus" ) +// env vars for user configuration +const ( + initConfigS3Prefix = "aws_fluent_bit_init_[sS]3" + initConfigFilePrefix = "aws_fluent_bit_init_[fF]ile" +) + // static paths const ( s3FileDirectoryPath = "/init/fluent-bit-init-s3-files/" @@ -100,7 +106,7 @@ func getECSTaskMetadata(httpClient HTTPClient) ECSTaskMetadata { metadata.ECS_TASK_DEFINITION = metadata.ECS_FAMILY + ":" + metadata.ECS_REVISION // per ECS task metadata docs, Cluster can be an ARN or the name - if (strings.Contains(metadata.ECS_CLUSTER, "/")) { + if strings.Contains(metadata.ECS_CLUSTER, "/") { clusterARN, err := arn.Parse(metadata.ECS_CLUSTER) if err != nil { logrus.Fatalf("[FluentBit Init Process] Failed to parse ECS Cluster ARN: %s %s\n", metadata.ECS_CLUSTER, err) @@ -169,8 +175,8 @@ func getAllConfigFiles() { envKey = string(env_kv[0]) envValue = string(env_kv[1]) - s3_regex, _ := regexp.Compile("aws_fluent_bit_init_[sS]3") - file_regex, _ := regexp.Compile("aws_fluent_bit_init_[fF]ile") + s3_regex, _ := regexp.Compile(initConfigS3Prefix) + file_regex, _ := regexp.Compile(initConfigFilePrefix) matched_s3 := s3_regex.MatchString(envKey) matched_file := file_regex.MatchString(envKey) From 4733358d560bfb2e83104e1498a7b8e29a4dd088 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Fri, 15 Mar 2024 11:50:05 -0700 Subject: [PATCH 2/5] init: make all env var matches case insensitive Signed-off-by: Wesley Pettit --- init/fluent_bit_init_process.go | 54 +++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/init/fluent_bit_init_process.go b/init/fluent_bit_init_process.go index c78418e74..ab66e4ada 100644 --- a/init/fluent_bit_init_process.go +++ b/init/fluent_bit_init_process.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "io" - "io/ioutil" "net/http" "os" "path/filepath" @@ -20,9 +19,11 @@ import ( ) // env vars for user configuration +// (?i) makes the match case insensitive const ( - initConfigS3Prefix = "aws_fluent_bit_init_[sS]3" - initConfigFilePrefix = "aws_fluent_bit_init_[fF]ile" + initS3ConfigFilePattern = "(?i)aws_fluent_bit_init_s3" + initLocalConfigFilePattern = "(?i)aws_fluent_bit_init_file" + initIgnoreFireLensConfig = "(?i)aws_fluent_bit_init_ignore_firelens" ) // static paths @@ -83,7 +84,7 @@ func getECSTaskMetadata(httpClient HTTPClient) ECSTaskMetadata { logrus.Fatalf("[FluentBit Init Process] Failed to get ECS Metadata via HTTP Get: %s\n", err) } - response, err := ioutil.ReadAll(res.Body) + response, err := io.ReadAll(res.Body) if err != nil { logrus.Fatalf("[FluentBit Init Process] Failed to read ECS Metadata from HTTP response: %s\n", err) } @@ -163,6 +164,9 @@ func getAllConfigFiles() { // get all env vars in the container envs := os.Environ() + s3Regex := regexp.MustCompile(initS3ConfigFilePattern) + fileRegex := regexp.MustCompile(initLocalConfigFilePattern) + // find all env vars match specified prefix for _, env := range envs { var envKey string @@ -175,27 +179,51 @@ func getAllConfigFiles() { envKey = string(env_kv[0]) envValue = string(env_kv[1]) - s3_regex, _ := regexp.Compile(initConfigS3Prefix) - file_regex, _ := regexp.Compile(initConfigFilePrefix) - - matched_s3 := s3_regex.MatchString(envKey) - matched_file := file_regex.MatchString(envKey) + matchedS3 := s3Regex.MatchString(envKey) + matchedFile := fileRegex.MatchString(envKey) // if this env var's value is an arn, download the config file first, then process it - if matched_s3 { + if matchedS3 { s3FilePath := getS3ConfigFile(envValue) s3FileName := strings.SplitN(s3FilePath, "/", -1) processConfigFile(s3FileDirectoryPath + s3FileName[len(s3FileName)-1]) } - // if this env var's value is a path of our built-in config file, process is derectly - if matched_file { + // if this env var's value is a local config fil, process is directly + if matchedFile { processConfigFile(envValue) } } } +func processFireLensConfigFile() { + includeFireLensConfig := true + envs := os.Environ() + + ignoreRegex := regexp.MustCompile(initIgnoreFireLensConfig) + + // docs say to use aws_fluent_bit_init_ignore_firelens + // this supports + for _, env := range envs { + var envKey string + var envValue string + env_kv := strings.SplitN(env, "=", 2) + if len(env_kv) != 2 { + logrus.Fatalf("[FluentBit Init Process] Unrecognizable environment variables: %s\n", env) + } + + envKey = string(env_kv[0]) + envValue = string(env_kv[1]) + + matchedIgnore := ignoreRegex.MatchString(envKey) + + if matchedIgnore { + includeFireLensConfig = false + } + } +} + func processConfigFile(path string) { - contentBytes, err := ioutil.ReadFile(path) + contentBytes, err := os.ReadFile(path) if err != nil { logrus.Errorln(err) logrus.Fatalf("[FluentBit Init Process] Cannot open file: %s\n", path) From 59060946355e0bf4df7624409d2c7984ea69c516 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Fri, 15 Mar 2024 13:51:10 -0700 Subject: [PATCH 3/5] init: add aws_fluent_bit_init_ignore_firelens to ignore the generated config Signed-off-by: Wesley Pettit --- init/fluent_bit_init_process.go | 44 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/init/fluent_bit_init_process.go b/init/fluent_bit_init_process.go index ab66e4ada..23cc3f993 100644 --- a/init/fluent_bit_init_process.go +++ b/init/fluent_bit_init_process.go @@ -28,10 +28,10 @@ const ( // static paths const ( - s3FileDirectoryPath = "/init/fluent-bit-init-s3-files/" - mainConfigFile = "/init/fluent-bit-init.conf" - originalMainConfigFile = "/fluent-bit/etc/fluent-bit.conf" - invokeFile = "/init/invoke_fluent_bit.sh" + s3FileDirectoryPath = "/init/fluent-bit-init-s3-files/" + initConfigFilePath = "/init/fluent-bit-init.conf" + firelensGeneratedConfigFilePath = "/fluent-bit/etc/fluent-bit.conf" + invokeFilePath = "/init/invoke_fluent_bit.sh" ) var ( @@ -205,14 +205,12 @@ func processFireLensConfigFile() { // this supports for _, env := range envs { var envKey string - var envValue string env_kv := strings.SplitN(env, "=", 2) if len(env_kv) != 2 { logrus.Fatalf("[FluentBit Init Process] Unrecognizable environment variables: %s\n", env) } envKey = string(env_kv[0]) - envValue = string(env_kv[1]) matchedIgnore := ignoreRegex.MatchString(envKey) @@ -220,6 +218,11 @@ func processFireLensConfigFile() { includeFireLensConfig = false } } + + if includeFireLensConfig { + // add @INCLUDE in main config file to include original main config file + writeInclude(firelensGeneratedConfigFilePath, initConfigFilePath) + } } func processConfigFile(path string) { @@ -236,7 +239,7 @@ func processConfigFile(path string) { updateCommand(path) } else { // this is not a parser config file. @INCLUDE - writeInclude(path, mainConfigFile) + writeInclude(path, initConfigFilePath) } } @@ -341,15 +344,15 @@ func downloadS3ConfigFile(s3Downloader S3Downloader, s3FilePath, bucketName, s3F } // use @INCLUDE to add config files to the main config file -func writeInclude(configFilePath, mainConfigFilePath string) { - mainConfigFile := openFile(mainConfigFilePath) - defer mainConfigFile.Close() +func writeInclude(configFilePath string, initConfigFilePath string) { + initConfigFile := openFile(initConfigFilePath) + defer initConfigFile.Close() writeContent := "@INCLUDE " + configFilePath + "\n" - _, err := mainConfigFile.WriteString(writeContent) + _, err := initConfigFile.WriteString(writeContent) if err != nil { logrus.Errorln(err) - logrus.Fatalf("[FluentBit Init Process] Cannot write %s in main config file: %s\n", writeContent[:len(writeContent)-2], mainConfigFilePath) + logrus.Fatalf("[FluentBit Init Process] Cannot write %s in main config file: %s\n", writeContent[:len(writeContent)-2], initConfigFilePath) } } @@ -406,23 +409,24 @@ func main() { // create the invoke_fluent_bit.sh // which will declare ECS Task Metadata as environment variables // and finally invoke Fluent Bit - createFile(invokeFile, true) + createFile(invokeFilePath, true) // get ECS Task Metadata and set the region for S3 client httpClient := &http.Client{} metadata := getECSTaskMetadata(httpClient) // set ECS Task Metada as env vars in the invoke_fluent_bit.sh - setECSTaskMetadata(metadata, invokeFile) + setECSTaskMetadata(metadata, invokeFilePath) // create main config file which will be used invoke Fluent Bit - createFile(mainConfigFile, true) - - // add @INCLUDE in main config file to include original main config file - writeInclude(originalMainConfigFile, mainConfigFile) + createFile(initConfigFilePath, true) // create Fluent Bit command to use "-c" to specify new main config file - createCommand(&baseCommand, mainConfigFile) + createCommand(&baseCommand, initConfigFilePath) + + // include the FireLens generated config + // unless the user has set aws_fluent_bit_init_ignore_firelens + processFireLensConfigFile() // get our built in config files or files from s3 // process built-in config files directly @@ -433,5 +437,5 @@ func main() { // this function will be called at the end // any error appear above will cause exit this process, // will not write Fluent Bit command in the finvoke_fluent_bit.sh so Fluent Bit will not be invoked - modifyInvokeFile(invokeFile) + modifyInvokeFile(invokeFilePath) } From cef8da007e3ea84cc7acbd97d4b8c6dc627a0fb6 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Fri, 15 Mar 2024 15:03:00 -0700 Subject: [PATCH 4/5] init: support on or true case insensitive for ignore_firelens Signed-off-by: Wesley Pettit --- init/fluent_bit_init_process.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/init/fluent_bit_init_process.go b/init/fluent_bit_init_process.go index 23cc3f993..90d93dee9 100644 --- a/init/fluent_bit_init_process.go +++ b/init/fluent_bit_init_process.go @@ -202,20 +202,25 @@ func processFireLensConfigFile() { ignoreRegex := regexp.MustCompile(initIgnoreFireLensConfig) // docs say to use aws_fluent_bit_init_ignore_firelens - // this supports + // this supports case insensitive prefix matching, in case someone + // tries to capitalize FireLens, or uses aws_fluent_bit_init_ignore_firelens_config for _, env := range envs { var envKey string + var envValue string env_kv := strings.SplitN(env, "=", 2) if len(env_kv) != 2 { logrus.Fatalf("[FluentBit Init Process] Unrecognizable environment variables: %s\n", env) } envKey = string(env_kv[0]) + envValue = string(env_kv[1]) matchedIgnore := ignoreRegex.MatchString(envKey) if matchedIgnore { - includeFireLensConfig = false + if strings.EqualFold(envValue, "true") || strings.EqualFold(envValue, "on") { + includeFireLensConfig = false + } } } From b11b821a658bbdc97f1f2708a5571ed44b4b0ab0 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Wed, 20 Mar 2024 11:04:01 -0700 Subject: [PATCH 5/5] init: add docs for init_ignore_firelens_config option Signed-off-by: Wesley Pettit --- .../init-process-for-fluent-bit/README.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/use_cases/init-process-for-fluent-bit/README.md b/use_cases/init-process-for-fluent-bit/README.md index d563ef79d..a63c398ec 100644 --- a/use_cases/init-process-for-fluent-bit/README.md +++ b/use_cases/init-process-for-fluent-bit/README.md @@ -138,6 +138,40 @@ You can use them as env vars directly in the Fluent Bit config. region ${AWS_REGION} ``` +### How to ignore the generated FireLens configuration? + +Please read the [Fluent Bit Out of Memory Kill Prevention Guide](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/oomkill-prevention#full-firelens-configuration-examples). It discusses settings to limit the memory usage of the Fluent Bit process and enable file buffering. These settings are specified on the input definition; in [FireLens the input configuration that collects stdout logs is autogenerated](https://aws.amazon.com/blogs/containers/under-the-hood-firelens-for-amazon-ecs-tasks/) by ECS. Therefore, enabling buffer control settings requires [overriding the generated FireLens input configuration](https://aws.amazon.com/blogs/containers/how-to-set-fluentd-and-fluent-bit-input-parameters-in-firelens/). + +The init process can help with this. Set the environment variable `aws_fluent_bit_init_ignore_firelens_config` on the container definition for the init image: + +``` + "environment": [ + { + "name": "aws_fluent_bit_init_ignore_firelens_config", + "value": "true" + }, + ], +``` + +And then make sure one of the configuration files specified for the init process contains an input definition like the following to collect stdout logs: + +``` +[INPUT] + Name forward + unix_path /var/run/fluent.sock + Mem_Buf_Limit 100MB +``` + +``` + "environment": [ + { + "name": "aws_fluent_bit_init_s3_1", + "value": "arn:aws:s3:::yourBucket/inputs.conf" + }, + ], +``` + +Please review the full detailed example on GitHub: [Overriding the FireLens input configuration using the init process](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/init-ignore-firelens) ### How init process works?