added fix for whitelisting k8s pods

lib/plugins/input/docker/docker.js

@@ -6,7 +6,15 @@ const consoleLogger = require('../../../util/logger.js')
 const ansiEscapeRegEx = /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g
 const dotRegex = /\./g
 const TRUE_REGEX = /true/i
-const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGSENE_ENABLED_DEFAULT || process.env.LOGS_ENABLED_DEFAULT || 'true')
+// The run.sh sets both LOGSENE_ENABLED_DEFAULT and LOGS_ENABLED_DEFAULT to TRUE by default.
+// Users should set either of these two to FALSE to disable logging by default for all containers.
+// This means you need to use whitelisting to enable logging for certain containers.
+// In the ENV for those containers set 'LOGS_ENABLED=true' or 'LOGSENE_ENABLED=true'.
+// const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test((process.env.LOGSENE_ENABLED_DEFAULT && process.env.LOGS_ENABLED_DEFAULT) || 'true')
+const LOGSENE_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGSENE_ENABLED_DEFAULT)
+const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGS_ENABLED_DEFAULT)
+const FINAL_LOGS_ENABLED_DEFAULT = LOGSENE_ENABLED_DEFAULT && LOGS_ENABLED_DEFAULT
+
 var ignoreLogsPattern = null
 var removeAnsiEscapeSeq = true
 var dockerInspectCache = {}
@@ -61,7 +69,7 @@ function InputDockerSocket (config, eventEmitter) {
       // filter via k8s annotations
       // in k8s we have to collect all logs and detach log streams
       // later once POD events are handled.
-      if (LOGS_ENABLED_DEFAULT === false &&
+      if (FINAL_LOGS_ENABLED_DEFAULT === false &&
           process.env.KUBERNETES_SERVICE_HOST !== undefined) { // LA running in k8s environment
         return true
       } else {

lib/plugins/input/docker/dockerInspect.js

@@ -9,7 +9,14 @@ var dockerInfo = {}
 var tagIds = null
 
 const TRUE_REGEX = /true/i
-const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGSENE_ENABLED_DEFAULT || process.env.LOGS_ENABLED_DEFAULT || 'true')
+// The run.sh sets both LOGSENE_ENABLED_DEFAULT and LOGS_ENABLED_DEFAULT to TRUE by default.
+// Users should set either of these two to FALSE to disable logging by default for all containers.
+// This means you need to use whitelisting to enable logging for certain containers.
+// In the ENV for those containers set 'LOGS_ENABLED=true' or 'LOGSENE_ENABLED=true'.
+// const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test((process.env.LOGSENE_ENABLED_DEFAULT && process.env.LOGS_ENABLED_DEFAULT) || 'true')
+const LOGSENE_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGSENE_ENABLED_DEFAULT)
+const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGS_ENABLED_DEFAULT)
+const FINAL_LOGS_ENABLED_DEFAULT = LOGSENE_ENABLED_DEFAULT && LOGS_ENABLED_DEFAULT
 
 docker.info(function dockerInfoHandler (err, data) {
   if (err) {
@@ -133,7 +140,7 @@ function getLogseneEnabled (info) {
     }
 
     if (info.LOGSENE_ENABLED === null) {
-      info.LOGSENE_ENABLED = LOGS_ENABLED_DEFAULT
+      info.LOGSENE_ENABLED = FINAL_LOGS_ENABLED_DEFAULT
     }
   }
 
@@ -196,7 +203,7 @@ function getLogseneToken (err, info) {
       // no Label or env var set, use LOGS_ENABLED_DEFAULT
       // console.log('Container ' + info.Id + ' ' + info.Name + ' setting LOGSENE_ENABLED not specified')
       // set the desired default from Logagent config environment
-      info.LOGSENE_ENABLED = LOGS_ENABLED_DEFAULT
+      info.LOGSENE_ENABLED = FINAL_LOGS_ENABLED_DEFAULT
     }
 
     if (info.LOGSENE_ENABLED === '0' || info.LOGSENE_ENABLED === 'false' || info.LOGSENE_ENABLED === 'no') {

lib/plugins/output-filter/kubernetes-enrichment.js

@@ -13,9 +13,18 @@ const podCache = new LRU({
 var client = null
 const FALSE_REGEX = /false/i
 const TRUE_REGEX = /true/i
-const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGSENE_ENABLED_DEFAULT || process.env.LOGS_ENABLED_DEFAULT || 'true')
 const useLogsEnabledPodAnnotation = TRUE_REGEX.test(process.env.USE_LOGS_ENABLED_K8S_ANNOTATIONS || 'true')
 
+// The run.sh sets both LOGSENE_ENABLED_DEFAULT and LOGS_ENABLED_DEFAULT to TRUE by default.
+// Users should set either of these two to FALSE to disable logging by default for all containers.
+// This means you need to use whitelisting to enable logging for certain containers.
+// In the ENV for those containers set 'LOGS_ENABLED=true' or 'LOGSENE_ENABLED=true'.
+// const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test((process.env.LOGSENE_ENABLED_DEFAULT && process.env.LOGS_ENABLED_DEFAULT) || 'true')
+
+const LOGSENE_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGSENE_ENABLED_DEFAULT)
+const LOGS_ENABLED_DEFAULT = TRUE_REGEX.test(process.env.LOGS_ENABLED_DEFAULT)
+const FINAL_LOGS_ENABLED_DEFAULT = LOGSENE_ENABLED_DEFAULT && LOGS_ENABLED_DEFAULT
+
 if (process.env.KUBERNETES_PORT_443_TCP !== undefined) {
   kubeconfig.loadFromCluster()
 } else {
@@ -72,12 +81,22 @@ function removeFields (pod, data) {
   }
 }
 
-function checkLogsEnabled (pod, data, context) {
+function checkLogsEnabled (pod, data, context, config) {
   if (pod.stLogEnabled !== undefined) {
     data.stLogEnabled = pod.stLogEnabled
+
+    if (config.debug === true) {
+      consoleLogger.log(`sematext.com/logs-enabled = ${pod.stLogEnabled} for ${pod.metadata.name}`)
+    }
+
     return
   }
-  pod.stLogEnabled = LOGS_ENABLED_DEFAULT
+
+  pod.stLogEnabled = FINAL_LOGS_ENABLED_DEFAULT
+
+  if (config.debug === true) {
+    consoleLogger.log(`sematext.com/logs-enabled = ${pod.stLogEnabled} for ${pod.metadata.name}`)
+  }
 
   var annotations = pod.metadata.annotations
   if (annotations) {
@@ -90,7 +109,10 @@ function checkLogsEnabled (pod, data, context) {
       }
     }
     data.stLogEnabled = pod.stLogEnabled
-    consoleLogger.log(`sematext.com/logs-enabled = ${pod.stLogEnabled} for ${pod.metadata.name}`)
+
+    if (config.debug === true) {
+      consoleLogger.log(`sematext.com/logs-enabled = ${pod.stLogEnabled} for ${pod.metadata.name}`)
+    }
   }
 }
 
@@ -139,10 +161,10 @@ function replaceDockerImageName (pod, data) {
   }
 }
 
-function processAnnotations (data, context, pod, eventEmitter, callback) {
+function processAnnotations (data, context, config, pod, eventEmitter, callback) {
   if (pod && pod.metadata) {
     if (useLogsEnabledPodAnnotation) {
-      checkLogsEnabled(pod, data, context)
+      checkLogsEnabled(pod, data, context, config)
       // var logsEnabled = context.dockerInspect.LOGSENE_ENABLED
       if (data.stLogEnabled === false) {
         // allow input plugins to close the input stream, e.g. input/docker/docker.js
@@ -169,7 +191,7 @@ function enrichLogs (context, config, eventEmitter, data, callback) {
   }
   const cachedPod = podCache.get(getPodCacheKey(data))
   if (cachedPod) {
-    processAnnotations(data, context, cachedPod, eventEmitter, callback)
+    processAnnotations(data, context, config, cachedPod, eventEmitter, callback)
     if (data.stLogEnabled === false) {
       if (config.debug === true) {
         consoleLogger.log('logs dropped ' + data.kubernetes.namespace + '/' + data.kubernetes.pod.name, data.message)

run.sh

@@ -28,7 +28,15 @@ fi
 echo $LA_CONFIG 
 
 export MAX_CLIENT_SOCKETS=${MAX_CLIENT_SOCKETS:-5}
+
+# Set both values to be default TRUE
+# Check in the code if they are both TRUE
+# If they are not both TRUE, because user sets at
+# least one of them to FALSE, the env will view 
+# them as FALSE
 export LOGSENE_ENABLED_DEFAULT=${LOGSENE_ENABLED_DEFAULT:-true}
+export LOGS_ENABLED_DEFAULT=${LOGS_ENABLED_DEFAULT:-true}
+
 export LOGSENE_TMP_DIR=/log-buffer
 mkdir -p $LOGSENE_TMP_DIR
 export LA_CONFIG_OVERRIDE=${LA_CONFIG_OVERRIDE:-false}