Introduce CLI flag -environment

CHANGELOG.next.asciidoc

@@ -39,8 +39,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
+- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
 - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568]
 - Fix missing output in dockerlogbeat {pull}15719[15719]
+- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442]
 
 *Auditbeat*
 

dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl

@@ -2,23 +2,25 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
- <key>Label</key>
+ <key>Label</key>
  <string>{{.identifier}}</string>
- <key>ProgramArguments</key>
- <array>
- <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
- <string>-c</string>
- <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
- <string>--path.home</string>
- <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
- <string>--path.config</string>
- <string>/etc/{{.BeatName}}</string>
- <string>--path.data</string>
- <string>/var/lib/{{.BeatName}}</string>
- <string>--path.logs</string>
- <string>/var/log/{{.BeatName}}</string>
- </array>
- <key>RunAtLoad</key>
- <true/>
+ <key>ProgramArguments</key>
+ <array>
+ <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
+ <string>-environment</string>
+ <string>macOS_service</string>
+ <string>-c</string>
+ <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
+ <string>--path.home</string>
+ <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
+ <string>--path.config</string>
+ <string>/etc/{{.BeatName}}</string>
+ <string>--path.data</string>
+ <string>/var/lib/{{.BeatName}}</string>
+ <string>--path.logs</string>
+ <string>/var/log/{{.BeatName}}</string>
+ </array>
+ <key>RunAtLoad</key>
+ <true/>
 </dict>
 </plist>

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -50,4 +50,4 @@ EXPOSE {{ $port }}
 
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
-CMD ["-e"]
+CMD ["-environment", "container"]

dev-tools/packaging/templates/linux/systemd.unit.tmpl

@@ -9,10 +9,10 @@ After=network-online.target
 User={{ .BeatUser }}
 Group={{ .BeatUser }}
 {{- end }}
-Environment="BEAT_LOG_OPTS=-e"
+Environment="BEAT_LOG_OPTS="
 Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml"
 Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}"
-ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
+ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} -environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
 Restart=always
 
 [Install]

dev-tools/packaging/templates/windows/install-service.ps1.tmpl

@@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path
 # Create the new service.
 New-Service -name {{.BeatName}} `
  -displayName {{.BeatName | title}} `
- -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
+ -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
 
 # Attempt to set the service to delayed start using sc config.
 Try {

libbeat/cmd/root.go

@@ -88,6 +88,7 @@ func GenRootCmdWithSettings(beatCreator beat.Creator, settings instance.Settings
  rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d"))
  rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v"))
  rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("e"))
+ rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("environment"))
  rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config"))
  rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data"))
  rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs"))

libbeat/docs/command-reference.asciidoc

@@ -910,6 +910,13 @@ messages.
 *`-e, --e`*::
 Logs to stderr and disables syslog/file output.
 
+*`-environment`*::
+For logging purposes, specifies the environment that {beatname_uc} is running in.
+This setting is used to select a default log output when no log output is configured.
+Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`.
+If `systemd` or `container` is specified, {beatname_uc} will log to stdout and stderr
+by default.
+
 *`--path.config`*::
 Sets the path for configuration files. See the <<directory-layout>> section for
 details.

libbeat/docs/shared-systemd.asciidoc

@@ -54,10 +54,6 @@ Logs are stored by default in journald. To view the Logs, use `journalctl`:
 journalctl -u {beatname_lc}.service
 ------------------------------------------------
 
-NOTE: The unit file included in the packages sets the `-e` flag by default.
-This flag makes {beatname_uc} log to stderr and disables other log outputs.
-Systemd stores all output sent to stderr in journald.
-
 [float]
 === Customize systemd unit for {beatname_uc}
 
@@ -67,7 +63,7 @@ override to change the default options.
 [cols="<h,<,<m",options="header",]
 |=======================================
 | Variable | Description | Default value
-| BEAT_LOG_OPTS | Log options | `-e`
+| BEAT_LOG_OPTS | Log options |
 | BEAT_CONFIG_OPTS | Flags for configuration file path | +-c /etc/{beatname_lc}/{beatname_lc}.yml+
 | BEAT_PATH_OPTS | Other paths | +-path.home /usr/share/{beatname_lc} -path.config /etc/{beatname_lc} -path.data /var/lib/{beatname_lc} -path.logs /var/log/{beatname_lc}+
 |=======================================
@@ -82,16 +78,7 @@ would override `BEAT_LOG_OPTS` to enable debug for Elasticsearch output.
 ["source", "systemd", subs="attributes"]
 ------------------------------------------------
 [Service]
-Environment="BEAT_LOG_OPTS=-e -d elasticsearch"
-------------------------------------------------
-
-To change the logging output from the {beatname_uc} configuration file, empty
-the environment variable. For example:
-
-["source", "systemd", subs="attributes"]
-------------------------------------------------
-[Service]
-Environment="BEAT_LOG_OPTS="
+Environment="BEAT_LOG_OPTS=-d elasticsearch"
 ------------------------------------------------
 
 To apply your changes, reload the systemd configuration and restart
@@ -109,4 +96,4 @@ include drop-in unit files. If you need to add a drop-in manually, use
 
 ifdef::apm-server[]
 include::./../config-ownership.asciidoc[]
-endif::apm-server[]
+endif::apm-server[]

libbeat/logp/config.go

@@ -17,7 +17,9 @@
 
 package logp
 
-import "time"
+import (
+ "time"
+)
 
 // Config contains the configuration options for the logger. To create a Config
 // from a common.Config use logp/config.Build.
@@ -52,20 +54,39 @@ type FileConfig struct {
  RedirectStderr bool `config:"redirect_stderr"`
 }
 
-var defaultConfig = Config{
- Level: InfoLevel,
- ToFiles: true,
- Files: FileConfig{
- MaxSize: 10 * 1024 * 1024,
- MaxBackups: 7,
- Permissions: 0600,
- Interval: 0,
- RotateOnStartup: true,
- },
- addCaller: true,
+// DefaultConfig returns the default config options for a given environment the
+// Beat is supposed to be run within.
+func DefaultConfig(environment Environment) Config {
+ switch environment {
+ case SystemdEnvironment, ContainerEnvironment:
+ return defaultToStderrConfig()
+
+ case MacOSServiceEnvironment, WindowsServiceEnvironment:
+ fallthrough
+ default:
+ return defaultToFileConfig()
+ }
+}
+
+func defaultToStderrConfig() Config {
+ return Config{
+ Level: InfoLevel,
+ ToStderr: true,
+ addCaller: true,
+ }
 }
 
-// DefaultConfig returns the default config options.
-func DefaultConfig() Config {
- return defaultConfig
+func defaultToFileConfig() Config {
+ return Config{
+ Level: InfoLevel,
+ ToFiles: true,
+ Files: FileConfig{
+ MaxSize: 10 * 1024 * 1024,
+ MaxBackups: 7,
+ Permissions: 0600,
+ Interval: 0,
+ RotateOnStartup: true,
+ },
+ addCaller: true,
+ }
 }

libbeat/logp/configure/logging.go

@@ -19,6 +19,7 @@ package configure
 
 import (
  "flag"
+ "fmt"
  "strings"
 
  "github.com/elastic/beats/libbeat/common"
@@ -30,18 +31,22 @@ var (
  verbose bool
  toStderr bool
  debugSelectors []string
+ environment logp.Environment
 )
 
+type environmentVar logp.Environment
+
 func init() {
  flag.BoolVar(&verbose, "v", false, "Log at INFO level")
  flag.BoolVar(&toStderr, "e", false, "Log to stderr and disable syslog/file output")
  common.StringArrVarFlag(nil, &debugSelectors, "d", "Enable certain debug selectors")
+ flag.Var((*environmentVar)(&environment), "environment", "set environment the Beat is run in")
 }
 
 // Logging builds a logp.Config based on the given common.Config and the specified
 // CLI flags.
 func Logging(beatName string, cfg *common.Config) error {
- config := logp.DefaultConfig()
+ config := logp.DefaultConfig(environment)
  config.Beat = beatName
  if cfg != nil {
  if err := cfg.Unpack(&config); err != nil {
@@ -69,3 +74,17 @@ func applyFlags(cfg *logp.Config) {
  cfg.Level = logp.DebugLevel
  }
 }
+
+func (v *environmentVar) Set(in string) error {
+ env := logp.ParseEnvironment(in)
+ if env == logp.InvalidEnvironment {
+ return fmt.Errorf("'%v' is not supported", in)
+ }
+
+ *(*logp.Environment)(v) = env
+ return nil
+}
+
+func (v *environmentVar) String() string {
+ return (*logp.Environment)(v).String()
+}

libbeat/logp/environment.go

@@ -0,0 +1,82 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package logp
+
+import "strings"
+
+// Environment indicates the environment the logger is supped to be run in.
+// The default logger configuration may be different for different environments.
+type Environment int
+
+const (
+ // DefaultEnvironment is used if the environment the process runs in is not known.
+ DefaultEnvironment Environment = iota
+
+ // SystemdEnvironment indicates that the process is started and managed by systemd.
+ SystemdEnvironment
+
+ // ContainerEnvironment indicates that the process is running within a container (docker, k8s, rkt, ...).
+ ContainerEnvironment
+
+ // MacOSServiceEnvironment indicates that the process is running as a daemon on macOS (e.g. managed via launchctl).
+ MacOSServiceEnvironment
+
+ // WindowsServiceEnvironment indicates the the process is run as a windows service.
+ WindowsServiceEnvironment
+
+ // InvalidEnvironment indicates that the environment name given is unknown or invalid.
+ InvalidEnvironment
+)
+
+// String returns the string representation the configured environment
+func (v Environment) String() string {
+ switch v {
+ case DefaultEnvironment:
+ return "default"
+ case SystemdEnvironment:
+ return "systemd"
+ case ContainerEnvironment:
+ return "container"
+ case MacOSServiceEnvironment:
+ return "macOS_service"
+ case WindowsServiceEnvironment:
+ return "windows_service"
+ default:
+ return "<invalid>"
+ }
+}
+
+// ParseEnvironment returns the environment type by name.
+// The parse is case insensitive.
+// InvalidEnvironment is returned if the environment type is unknown.
+func ParseEnvironment(in string) Environment {
+ switch strings.ToLower(in) {
+ case "default":
+ return DefaultEnvironment
+ case "systemd":
+ return SystemdEnvironment
+ case "container":
+ return ContainerEnvironment
+ case "macos_service":
+ return MacOSServiceEnvironment
+ case "windows_service":
+ return WindowsServiceEnvironment
+ default:
+ return InvalidEnvironment
+ }
+}