From 2698bc1af62f7a675b17d7d2cf800253324da9aa Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 9 Jan 2020 16:24:40 +0100 Subject: [PATCH 01/15] Introduce CLI flag -environment Introduce CLI flags -environment to control the default logging settings the Beat should use if no logging is configured. The behavior of -e does not change. By replacing `-e` with `-environment system` in the system unit file we continue to log to stdout/stderr by default, but users are still able to overwrite settings. --- .../templates/linux/systemd.unit.tmpl | 2 +- libbeat/logp/config.go | 48 +++++++++++++------ libbeat/logp/configure/logging.go | 21 +++++++- libbeat/logp/environment.go | 48 +++++++++++++++++++ 4 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 libbeat/logp/environment.go diff --git a/dev-tools/packaging/templates/linux/systemd.unit.tmpl b/dev-tools/packaging/templates/linux/systemd.unit.tmpl index 7f27459b465..e72bc28a86f 100644 --- a/dev-tools/packaging/templates/linux/systemd.unit.tmpl +++ b/dev-tools/packaging/templates/linux/systemd.unit.tmpl @@ -9,7 +9,7 @@ After=network-online.target User={{ .BeatUser }} Group={{ .BeatUser }} {{- end }} -Environment="BEAT_LOG_OPTS=-e" +Environment="BEAT_LOG_OPTS=-environment systemd" Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml" Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}" ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index 245dee5e60a..e3fe0f5a891 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -17,7 +17,9 @@ package logp -import "time" +import ( + "time" +) // Config contains the configuration options for the logger. To create a Config // from a common.Config use logp/config.Build. @@ -52,20 +54,36 @@ type FileConfig struct { RedirectStderr bool `config:"redirect_stderr"` } -var defaultConfig = Config{ - Level: InfoLevel, - ToFiles: true, - Files: FileConfig{ - MaxSize: 10 * 1024 * 1024, - MaxBackups: 7, - Permissions: 0600, - Interval: 0, - RotateOnStartup: true, - }, - addCaller: true, +// DefaultConfig returns the default config options for a given environment the +// Beat is supposed to be run within. +func DefaultConfig(environment Environment) Config { + switch environment { + case SystemdEnvironment: + return defaultSystemdConfig() + default: + return defaultEnvConfig() + } } -// DefaultConfig returns the default config options. -func DefaultConfig() Config { - return defaultConfig +func defaultSystemdConfig() Config { + return Config{ + Level: InfoLevel, + ToStderr: true, + addCaller: true, + } +} + +func defaultEnvConfig() Config { + return Config{ + Level: InfoLevel, + ToFiles: true, + Files: FileConfig{ + MaxSize: 10 * 1024 * 1024, + MaxBackups: 7, + Permissions: 0600, + Interval: 0, + RotateOnStartup: true, + }, + addCaller: true, + } } diff --git a/libbeat/logp/configure/logging.go b/libbeat/logp/configure/logging.go index c57acd4b39c..be1a7c5c6ab 100644 --- a/libbeat/logp/configure/logging.go +++ b/libbeat/logp/configure/logging.go @@ -19,6 +19,7 @@ package configure import ( "flag" + "fmt" "strings" "github.com/elastic/beats/libbeat/common" @@ -30,18 +31,22 @@ var ( verbose bool toStderr bool debugSelectors []string + environment logp.Environment ) +type environmentVar logp.Environment + func init() { flag.BoolVar(&verbose, "v", false, "Log at INFO level") flag.BoolVar(&toStderr, "e", false, "Log to stderr and disable syslog/file output") common.StringArrVarFlag(nil, &debugSelectors, "d", "Enable certain debug selectors") + flag.Var((*environmentVar)(&environment), "environment", "set environment the Beat is run in") } // Logging builds a logp.Config based on the given common.Config and the specified // CLI flags. func Logging(beatName string, cfg *common.Config) error { - config := logp.DefaultConfig() + config := logp.DefaultConfig(environment) config.Beat = beatName if cfg != nil { if err := cfg.Unpack(&config); err != nil { @@ -69,3 +74,17 @@ func applyFlags(cfg *logp.Config) { cfg.Level = logp.DebugLevel } } + +func (v *environmentVar) Set(in string) error { + env := logp.ParseEnvironment(in) + if env == logp.InvalidEnvironment { + return fmt.Errorf("'%v' is not supported", in) + } + + *(*logp.Environment)(v) = env + return nil +} + +func (v *environmentVar) String() string { + return (*logp.Environment)(v).String() +} diff --git a/libbeat/logp/environment.go b/libbeat/logp/environment.go new file mode 100644 index 00000000000..76cf59a2c8f --- /dev/null +++ b/libbeat/logp/environment.go @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package logp + +type Environment int + +const ( + DefaultEnvironment Environment = iota + SystemdEnvironment + InvalidEnvironment +) + +func (v Environment) String() string { + switch v { + case DefaultEnvironment: + return "default" + case SystemdEnvironment: + return "systemd" + default: + return "" + } +} + +func ParseEnvironment(in string) Environment { + switch in { + case "default": + return DefaultEnvironment + case "systemd": + return SystemdEnvironment + default: + return InvalidEnvironment + } +} From 8d38fa1a159541bc97373d392035561d61c1f6a8 Mon Sep 17 00:00:00 2001 From: urso Date: Wed, 15 Jan 2020 20:51:00 +0100 Subject: [PATCH 02/15] Add more environment types Add windows_service, macos_service, and container envionment types. If beats are installed using our scripts the `-environment` flag will always be set. Replace `-e` with `-environment docker` CLI flag in the Dockerfile. Although it's uncommon, users can not overwrite the logging configuration in docker containers as well. The docker(container) environment and systemd environment will default to stdout/stderr logging. All other environments continue to use file rotation as default. --- .../darwin/launchd-daemon.plist.tmpl | 2 ++ .../templates/docker/Dockerfile.tmpl | 2 +- .../windows/install-service.ps1.tmpl | 2 +- libbeat/logp/config.go | 9 ++++++--- libbeat/logp/environment.go | 19 ++++++++++++++++++- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl index 683188534a3..a293ea386a2 100644 --- a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl +++ b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl @@ -7,6 +7,8 @@ ProgramArguments {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} + --environment + macOS_service -c /etc/{{.BeatName}}/{{.BeatName}}.yml --path.home diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 0fae61cdbdf..a92448f1c11 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -50,4 +50,4 @@ EXPOSE {{ $port }} WORKDIR {{ $beatHome }} ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] -CMD ["-e"] +CMD ["-environment", "docker"] diff --git a/dev-tools/packaging/templates/windows/install-service.ps1.tmpl b/dev-tools/packaging/templates/windows/install-service.ps1.tmpl index cab1373ef15..c398e18d333 100644 --- a/dev-tools/packaging/templates/windows/install-service.ps1.tmpl +++ b/dev-tools/packaging/templates/windows/install-service.ps1.tmpl @@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path # Create the new service. New-Service -name {{.BeatName}} ` -displayName {{.BeatName | title}} ` - -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true" + -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true" # Attempt to set the service to delayed start using sc config. Try { diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index e3fe0f5a891..40a9582161a 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -58,10 +58,13 @@ type FileConfig struct { // Beat is supposed to be run within. func DefaultConfig(environment Environment) Config { switch environment { - case SystemdEnvironment: + case SystemdEnvironment, ContainerEnvironment: return defaultSystemdConfig() + + case MacOSServiceEnvironment, WindowsServiceEnvironment: + fallthrough default: - return defaultEnvConfig() + return defaultToFileConfig() } } @@ -73,7 +76,7 @@ func defaultSystemdConfig() Config { } } -func defaultEnvConfig() Config { +func defaultToFileConfig() Config { return Config{ Level: InfoLevel, ToFiles: true, diff --git a/libbeat/logp/environment.go b/libbeat/logp/environment.go index 76cf59a2c8f..36e42c79de1 100644 --- a/libbeat/logp/environment.go +++ b/libbeat/logp/environment.go @@ -17,11 +17,16 @@ package logp +import "strings" + type Environment int const ( DefaultEnvironment Environment = iota SystemdEnvironment + ContainerEnvironment + MacOSServiceEnvironment + WindowsServiceEnvironment InvalidEnvironment ) @@ -31,17 +36,29 @@ func (v Environment) String() string { return "default" case SystemdEnvironment: return "systemd" + case ContainerEnvironment: + return "container" + case MacOSServiceEnvironment: + return "macOS_service" + case WindowsServiceEnvironment: + return "windows_service" default: return "" } } func ParseEnvironment(in string) Environment { - switch in { + switch strings.ToLower(in) { case "default": return DefaultEnvironment case "systemd": return SystemdEnvironment + case "docker", "container", "kubernetes", "k8s": + return ContainerEnvironment + case "macos_service": + return MacOSServiceEnvironment + case "windows_service": + return WindowsServiceEnvironment default: return InvalidEnvironment } From 36d50852e271b3d1dcbea2a022f15fb3b04c4d94 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 16:02:14 +0100 Subject: [PATCH 03/15] always pass -environment=systemd --- .../packaging/templates/darwin/launchd-daemon.plist.tmpl | 2 +- dev-tools/packaging/templates/linux/systemd.unit.tmpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl index a293ea386a2..a33dc0ec80e 100644 --- a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl +++ b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl @@ -7,7 +7,7 @@ ProgramArguments {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} - --environment + -environment macOS_service -c /etc/{{.BeatName}}/{{.BeatName}}.yml diff --git a/dev-tools/packaging/templates/linux/systemd.unit.tmpl b/dev-tools/packaging/templates/linux/systemd.unit.tmpl index e72bc28a86f..10a56c0b590 100644 --- a/dev-tools/packaging/templates/linux/systemd.unit.tmpl +++ b/dev-tools/packaging/templates/linux/systemd.unit.tmpl @@ -9,10 +9,10 @@ After=network-online.target User={{ .BeatUser }} Group={{ .BeatUser }} {{- end }} -Environment="BEAT_LOG_OPTS=-environment systemd" +Environment="BEAT_LOG_OPTS=" Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml" Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}" -ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS +ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} -environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS Restart=always [Install] From fd0cc10944da27a0c129c07c3a4511570224caab Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 16:04:09 +0100 Subject: [PATCH 04/15] Add flag to command root --- libbeat/cmd/root.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index e76a7ae9d90..94382eed5ba 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -88,6 +88,7 @@ func GenRootCmdWithSettings(beatCreator beat.Creator, settings instance.Settings rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("e")) + rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("environment")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs")) From 31cdfddd2a15818e0db2d72e6f0fccdb0648223e Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 16:08:16 +0100 Subject: [PATCH 05/15] Update docs --- libbeat/docs/shared-systemd.asciidoc | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/libbeat/docs/shared-systemd.asciidoc b/libbeat/docs/shared-systemd.asciidoc index 841a2f7707f..7f80a463edb 100644 --- a/libbeat/docs/shared-systemd.asciidoc +++ b/libbeat/docs/shared-systemd.asciidoc @@ -54,10 +54,6 @@ Logs are stored by default in journald. To view the Logs, use `journalctl`: journalctl -u {beatname_lc}.service ------------------------------------------------ -NOTE: The unit file included in the packages sets the `-e` flag by default. -This flag makes {beatname_uc} log to stderr and disables other log outputs. -Systemd stores all output sent to stderr in journald. - [float] === Customize systemd unit for {beatname_uc} @@ -67,7 +63,7 @@ override to change the default options. [cols=" Date: Thu, 16 Jan 2020 16:19:42 +0100 Subject: [PATCH 06/15] Add changelog --- CHANGELOG.next.asciidoc | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1515706df76..3152de5f2e3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -41,6 +41,59 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568] - Fix missing output in dockerlogbeat {pull}15719[15719] +- Make the behavior of clientWorker and netClientWorker consistent when error is returned from publisher pipeline +- Fix a bug, publisher pipeline exits if output returns an error, irrespective of pipeline is closed or not +- Fix typo in TLS renegotiation configuration and setting the option correctly {issue}10871[10871], {pull}12354[12354] +- Ensure all beat commands respect configured settings. {pull}10721[10721] +- Add missing fields and test cases for libbeat add_kubernetes_metadata processor. {issue}11133[11133], {pull}11134[11134] +- decode_json_field: process objects and arrays only {pull}11312[11312] +- decode_json_field: do not process arrays when flag not set. {pull}11318[11318] +- Report faulting file when config reload fails. {pull}11304[11304] +- Fix a typo in libbeat/outputs/transport/client.go by updating `c.conn.LocalAddr()` to `c.conn.RemoteAddr()`. {pull}11242[11242] +- Management configuration backup file will now have a timestamps in their name. {pull}11034[11034] +- [CM] Parse enrollment_token response correctly {pull}11648[11648] +- Not hiding error in case of http failure using elastic fetcher {pull}11604[11604] +- Escape BOM on JsonReader before trying to decode line {pull}11661[11661] +- Fix matching of string arrays in contains condition. {pull}11691[11691] +- Replace wmi queries with win32 api calls as they were consuming CPU resources {issue}3249[3249] and {issue}11840[11840] +- Fix a race condition with the Kafka pipeline client, it is possible that `Close()` get called before `Connect()` . {issue}11945[11945] +- Fix queue.spool.write.flush.events config type. {pull}12080[12080] +- Fixed a memory leak when using the add_process_metadata processor under Windows. {pull}12100[12100] +- Fix of docker json parser for missing "log" jsonkey in docker container's log {issue}11464[11464] +- Fixed Beat ID being reported by GET / API. {pull}12180[12180] +- Fixed setting bulk max size in kafka output. {pull}12254[12254] +- Add host.os.codename to fields.yml. {pull}12261[12261] +- Fix `@timestamp` being duplicated in events if `@timestamp` is set in a + processor (or by any code utilizing `PutValue()` on a `beat.Event`). +- Fix leak in script processor when using Javascript functions in a processor chain. {pull}12600[12600] +- Add additional nil pointer checks to Docker client code to deal with vSphere Integrated Containers {pull}12628[12628] +- Fixed `json.add_error_key` property setting for delivering error messages from beat events {pull}11298[11298] +- Fix Central Management enroll under Windows {issue}12797[12797] {pull}12799[12799] +- ILM: Use GET instead of HEAD when checking for alias to expose detailed error message. {pull}12886[12886] +- Fix seccomp policy preventing some features to function properly on 32bit Linux systems. {issue}12990[12990] {pull}13008[13008] +- Fix unexpected stops on docker autodiscover when a container is restarted before `cleanup_timeout`. {issue}12962[12962] {pull}13127[13127] +- Fix install-service.ps1's ability to set Windows service's delay start configuration. {pull}13173[13173] +- Fix some incorrect types and formats in field.yml files. {pull}13188[13188] +- Load DLLs only from Windows system directory. {pull}13234[13234] {pull}13384[13384] +- Fix mapping for kubernetes.labels and kubernetes.annotations in add_kubernetes_metadata. {issue}12638[12638] {pull}13226[13226] +- Fix case insensitive regular expressions not working correctly. {pull}13250[13250] +- Disable `add_kubernetes_metadata` if no matchers found. {pull}13709[13709] +- Better wording for xpack beats when the _xpack endpoint is not reachable. {pull}13771[13771] +- Recover from panics in the javascript process and log details about the failure to aid in future debugging. {pull}13690[13690] +- Make the script processor concurrency-safe. {issue}13690[13690] {pull}13857[13857] +- Kubernetes watcher at `add_kubernetes_metadata` fails with StatefulSets {pull}13905[13905] +- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over + TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- Support usage of custom builders without hints and mappers {pull}13839[13839] +- Fix memory leak in kubernetes autodiscover provider and add_kubernetes_metadata processor happening when pods are terminated without sending a delete event. {pull}14259[14259] +- Fix kubernetes `metaGenerator.ResourceMetadata` when parent reference controller is nil {issue}14320[14320] {pull}14329[14329] +- Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338] +- Fix `proxy_url` option in Elasticsearch output. {pull}14950[14950] +- Fix bug with potential concurrent reads and writes from event.Meta map by Kafka output. {issue}14542[14542] {pull}14568[14568] +- Fix spooling to disk blocking infinitely if the lock file can not be acquired. {pull}15338[15338] +- Fix `metricbeat test output` with an ipv6 ES host in the output.hosts. {pull}15368[15368] +- Fix `convert` processor conversion of string to integer with leading zeros. {issue}15513[15513] {pull}15557[15557] +- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442] *Auditbeat* From fc556febeb3d0c74cdbb033b620ebf398f051bfc Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 16:32:55 +0100 Subject: [PATCH 07/15] Feed the hound --- libbeat/logp/environment.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libbeat/logp/environment.go b/libbeat/logp/environment.go index 36e42c79de1..ee1eb49ac9c 100644 --- a/libbeat/logp/environment.go +++ b/libbeat/logp/environment.go @@ -19,17 +19,31 @@ package logp import "strings" +// Environment indicates the environment the logger is supped to be run in. +// The default logger configuration may be different for different environments. type Environment int const ( + // DefaultEnvironment is used if the environment the process runs in is not known. DefaultEnvironment Environment = iota + + // SystemdEnvironment indicates that the process is started and managed by systemd. SystemdEnvironment + + // ContainerEnvironment indicates that the process is running within a container (docker, k8s, rkt, ...). ContainerEnvironment + + // MacOSServiceEnvironment indicates that the process is running as a daemon on macOS (e.g. managed via launchctl). MacOSServiceEnvironment + + // WindowsServiceEnvironment indicates the the process is run as a windows service. WindowsServiceEnvironment + + // InvalidEnvironment indicates that the environment name given is unknown or invalid. InvalidEnvironment ) +// String returns the string representation the configured environment func (v Environment) String() string { switch v { case DefaultEnvironment: @@ -47,6 +61,9 @@ func (v Environment) String() string { } } +// ParseEnvironment returns the environment type by name. +// The parse is case insensitive. +// InvalidEnvironment is returned if the environment type is unknown. func ParseEnvironment(in string) Environment { switch strings.ToLower(in) { case "default": From 00d9a5f66de33f2316d7adc8207cffc12b62e3e9 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 17:47:36 +0100 Subject: [PATCH 08/15] review --- .../templates/darwin/launchd-daemon.plist.tmpl | 12 ++++++------ dev-tools/packaging/templates/docker/Dockerfile.tmpl | 2 +- libbeat/logp/config.go | 4 ++-- libbeat/logp/environment.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl index a33dc0ec80e..235524aacb5 100644 --- a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl +++ b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl @@ -2,13 +2,13 @@ - Label + Label {{.identifier}} - ProgramArguments - + ProgramArguments + {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} - -environment - macOS_service + -environment + macOS_service -c /etc/{{.BeatName}}/{{.BeatName}}.yml --path.home @@ -20,7 +20,7 @@ --path.logs /var/log/{{.BeatName}} - RunAtLoad + RunAtLoad diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index a92448f1c11..bd5eb0f3c1a 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -50,4 +50,4 @@ EXPOSE {{ $port }} WORKDIR {{ $beatHome }} ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] -CMD ["-environment", "docker"] +CMD ["-environment", "container"] diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index 40a9582161a..b04ef2e6a23 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -59,7 +59,7 @@ type FileConfig struct { func DefaultConfig(environment Environment) Config { switch environment { case SystemdEnvironment, ContainerEnvironment: - return defaultSystemdConfig() + return defaultToStderrConfig() case MacOSServiceEnvironment, WindowsServiceEnvironment: fallthrough @@ -68,7 +68,7 @@ func DefaultConfig(environment Environment) Config { } } -func defaultSystemdConfig() Config { +func defaultToStderrConfig() Config { return Config{ Level: InfoLevel, ToStderr: true, diff --git a/libbeat/logp/environment.go b/libbeat/logp/environment.go index ee1eb49ac9c..ae74ba7d46b 100644 --- a/libbeat/logp/environment.go +++ b/libbeat/logp/environment.go @@ -70,7 +70,7 @@ func ParseEnvironment(in string) Environment { return DefaultEnvironment case "systemd": return SystemdEnvironment - case "docker", "container", "kubernetes", "k8s": + case "container": return ContainerEnvironment case "macos_service": return MacOSServiceEnvironment From 744f3a03ecbbb70a3510da1e6884aeb12814d49a Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 17:53:30 +0100 Subject: [PATCH 09/15] Document -environment CLI flag --- libbeat/docs/command-reference.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index 42a38af159f..a830c8222a5 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -910,6 +910,13 @@ messages. *`-e, --e`*:: Logs to stderr and disables syslog/file output. +*`-environment`*:: +Tell {beatname_uc} the environment it runs into. If no log output is configured +the Beat will select the default output based on the environment. +Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`. +{beatname_uc} will log to stdout and stderr by default if the `systemd` or +`container` environment is configured. + *`--path.config`*:: Sets the path for configuration files. See the <> section for details. From 3a44428b5886a23e627a6a58891fd6c869496c79 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 17:57:07 +0100 Subject: [PATCH 10/15] unify indentation to 4 spaces --- .../darwin/launchd-daemon.plist.tmpl | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl index 235524aacb5..b98e9081172 100644 --- a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl +++ b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl @@ -2,25 +2,25 @@ - Label + Label {{.identifier}} - ProgramArguments - - {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} - -environment - macOS_service - -c - /etc/{{.BeatName}}/{{.BeatName}}.yml - --path.home - {{.install_path}}/{{.BeatVendor}}/{{.BeatName}} - --path.config - /etc/{{.BeatName}} - --path.data - /var/lib/{{.BeatName}} - --path.logs - /var/log/{{.BeatName}} - - RunAtLoad - + ProgramArguments + + {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} + -environment + macOS_service + -c + /etc/{{.BeatName}}/{{.BeatName}}.yml + --path.home + {{.install_path}}/{{.BeatVendor}}/{{.BeatName}} + --path.config + /etc/{{.BeatName}} + --path.data + /var/lib/{{.BeatName}} + --path.logs + /var/log/{{.BeatName}} + + RunAtLoad + From 1d6b502095f3311d0a29b20d0267950f575df188 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 16 Jan 2020 22:11:02 +0100 Subject: [PATCH 11/15] fix changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3152de5f2e3..255b56c5552 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -93,6 +93,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix spooling to disk blocking infinitely if the lock file can not be acquired. {pull}15338[15338] - Fix `metricbeat test output` with an ipv6 ES host in the output.hosts. {pull}15368[15368] - Fix `convert` processor conversion of string to integer with leading zeros. {issue}15513[15513] {pull}15557[15557] +- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442] *Auditbeat* From 1aa6c1e727ec75b19603672a3b39099589084a61 Mon Sep 17 00:00:00 2001 From: urso Date: Sat, 18 Jan 2020 01:59:54 +0100 Subject: [PATCH 12/15] Fix changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 255b56c5552..768ef47f32b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -94,6 +94,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `metricbeat test output` with an ipv6 ES host in the output.hosts. {pull}15368[15368] - Fix `convert` processor conversion of string to integer with leading zeros. {issue}15513[15513] {pull}15557[15557] - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442] *Auditbeat* From c070704d265ba812e9223f5ec4c06ad0bd5540ba Mon Sep 17 00:00:00 2001 From: urso Date: Sat, 18 Jan 2020 02:00:03 +0100 Subject: [PATCH 13/15] Update docs --- libbeat/docs/command-reference.asciidoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index a830c8222a5..0f6042e986f 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -911,11 +911,11 @@ messages. Logs to stderr and disables syslog/file output. *`-environment`*:: -Tell {beatname_uc} the environment it runs into. If no log output is configured -the Beat will select the default output based on the environment. +For logging purposes, specifies the environment that {beatname_uc} is running in. +This setting is used to select a default log output when no log output is configured. Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`. -{beatname_uc} will log to stdout and stderr by default if the `systemd` or -`container` environment is configured. +If `systemd` or `container` is specified, {beatname_uc} will log to stdout and stderr +by default. *`--path.config`*:: Sets the path for configuration files. See the <> section for From 3567069ce00a757f6f1cb2afda33947c3e32c0d2 Mon Sep 17 00:00:00 2001 From: urso Date: Mon, 20 Jan 2020 14:50:16 +0100 Subject: [PATCH 14/15] hmm... --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 768ef47f32b..aebe2e6bae5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -39,6 +39,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568] - Fix missing output in dockerlogbeat {pull}15719[15719] - Make the behavior of clientWorker and netClientWorker consistent when error is returned from publisher pipeline From 64aff4c1a14fcf5199a8570f7e507837e23679a6 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 23 Jan 2020 17:51:24 +0100 Subject: [PATCH 15/15] fix changelog... again --- CHANGELOG.next.asciidoc | 54 ----------------------------------------- 1 file changed, 54 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index aebe2e6bae5..c334a6b28a6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -42,60 +42,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568] - Fix missing output in dockerlogbeat {pull}15719[15719] -- Make the behavior of clientWorker and netClientWorker consistent when error is returned from publisher pipeline -- Fix a bug, publisher pipeline exits if output returns an error, irrespective of pipeline is closed or not -- Fix typo in TLS renegotiation configuration and setting the option correctly {issue}10871[10871], {pull}12354[12354] -- Ensure all beat commands respect configured settings. {pull}10721[10721] -- Add missing fields and test cases for libbeat add_kubernetes_metadata processor. {issue}11133[11133], {pull}11134[11134] -- decode_json_field: process objects and arrays only {pull}11312[11312] -- decode_json_field: do not process arrays when flag not set. {pull}11318[11318] -- Report faulting file when config reload fails. {pull}11304[11304] -- Fix a typo in libbeat/outputs/transport/client.go by updating `c.conn.LocalAddr()` to `c.conn.RemoteAddr()`. {pull}11242[11242] -- Management configuration backup file will now have a timestamps in their name. {pull}11034[11034] -- [CM] Parse enrollment_token response correctly {pull}11648[11648] -- Not hiding error in case of http failure using elastic fetcher {pull}11604[11604] -- Escape BOM on JsonReader before trying to decode line {pull}11661[11661] -- Fix matching of string arrays in contains condition. {pull}11691[11691] -- Replace wmi queries with win32 api calls as they were consuming CPU resources {issue}3249[3249] and {issue}11840[11840] -- Fix a race condition with the Kafka pipeline client, it is possible that `Close()` get called before `Connect()` . {issue}11945[11945] -- Fix queue.spool.write.flush.events config type. {pull}12080[12080] -- Fixed a memory leak when using the add_process_metadata processor under Windows. {pull}12100[12100] -- Fix of docker json parser for missing "log" jsonkey in docker container's log {issue}11464[11464] -- Fixed Beat ID being reported by GET / API. {pull}12180[12180] -- Fixed setting bulk max size in kafka output. {pull}12254[12254] -- Add host.os.codename to fields.yml. {pull}12261[12261] -- Fix `@timestamp` being duplicated in events if `@timestamp` is set in a - processor (or by any code utilizing `PutValue()` on a `beat.Event`). -- Fix leak in script processor when using Javascript functions in a processor chain. {pull}12600[12600] -- Add additional nil pointer checks to Docker client code to deal with vSphere Integrated Containers {pull}12628[12628] -- Fixed `json.add_error_key` property setting for delivering error messages from beat events {pull}11298[11298] -- Fix Central Management enroll under Windows {issue}12797[12797] {pull}12799[12799] -- ILM: Use GET instead of HEAD when checking for alias to expose detailed error message. {pull}12886[12886] -- Fix seccomp policy preventing some features to function properly on 32bit Linux systems. {issue}12990[12990] {pull}13008[13008] -- Fix unexpected stops on docker autodiscover when a container is restarted before `cleanup_timeout`. {issue}12962[12962] {pull}13127[13127] -- Fix install-service.ps1's ability to set Windows service's delay start configuration. {pull}13173[13173] -- Fix some incorrect types and formats in field.yml files. {pull}13188[13188] -- Load DLLs only from Windows system directory. {pull}13234[13234] {pull}13384[13384] -- Fix mapping for kubernetes.labels and kubernetes.annotations in add_kubernetes_metadata. {issue}12638[12638] {pull}13226[13226] -- Fix case insensitive regular expressions not working correctly. {pull}13250[13250] -- Disable `add_kubernetes_metadata` if no matchers found. {pull}13709[13709] -- Better wording for xpack beats when the _xpack endpoint is not reachable. {pull}13771[13771] -- Recover from panics in the javascript process and log details about the failure to aid in future debugging. {pull}13690[13690] -- Make the script processor concurrency-safe. {issue}13690[13690] {pull}13857[13857] -- Kubernetes watcher at `add_kubernetes_metadata` fails with StatefulSets {pull}13905[13905] -- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] -- Support usage of custom builders without hints and mappers {pull}13839[13839] -- Fix memory leak in kubernetes autodiscover provider and add_kubernetes_metadata processor happening when pods are terminated without sending a delete event. {pull}14259[14259] -- Fix kubernetes `metaGenerator.ResourceMetadata` when parent reference controller is nil {issue}14320[14320] {pull}14329[14329] -- Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338] -- Fix `proxy_url` option in Elasticsearch output. {pull}14950[14950] -- Fix bug with potential concurrent reads and writes from event.Meta map by Kafka output. {issue}14542[14542] {pull}14568[14568] -- Fix spooling to disk blocking infinitely if the lock file can not be acquired. {pull}15338[15338] -- Fix `metricbeat test output` with an ipv6 ES host in the output.hosts. {pull}15368[15368] -- Fix `convert` processor conversion of string to integer with leading zeros. {issue}15513[15513] {pull}15557[15557] -- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] -- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442] *Auditbeat*