Add ECS logging MVP to libbeat (#17974)

Enable minimal ECS conformant logging by setting `logging.ecs:true`:
* adds @timestamp, log.level, message and ecs.version to every log line
* if EnableCaller is set add log.origin.file and log.origin.line.
* logs error.message when error is given

implements #17942

CHANGELOG.next.asciidoc

@@ -274,6 +274,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add support for basic ECS logging. {pull}17974[17974]
 - Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153]
 - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 

NOTICE.txt

@@ -3263,7 +3263,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: github.com/hashicorp/errwrap
-Revision: 7554cd9344ce
+Version: v1.0.0
 License type (autodetected): MPL-2.0
 ./vendor/github.com/hashicorp/errwrap/LICENSE:
 --------------------------------------------------------------------
@@ -3624,7 +3624,7 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice
 
 --------------------------------------------------------------------
 Dependency: github.com/hashicorp/go-multierror
-Revision: ed905158d874
+Version: v1.0.0
 License type (autodetected): MPL-2.0
 ./vendor/github.com/hashicorp/go-multierror/LICENSE:
 --------------------------------------------------------------------
@@ -7179,7 +7179,7 @@ SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: github.com/stretchr/testify
-Version: v1.4.0
+Version: v1.5.0
 License type (autodetected): MIT
 ./vendor/github.com/stretchr/testify/LICENSE:
 --------------------------------------------------------------------
@@ -7519,6 +7519,117 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: go.elastic.co/ecszap
+Version: v0.1.1
+Revision: cdd95a104193
+License type (autodetected): Apache-2.0
+./vendor/go.elastic.co/ecszap/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+-------NOTICE.txt-----
+ecszap
+Copyright 2020-2020 Elasticsearch B.V.
+
+==========================================================================
+Third party libraries used by the Elastic ecszap project:
+==========================================================================
+
+Dependency: github.com/stretchr/testify
+Version: v1.4.0
+License type: MIT
+https://github.com/stretchr/testify/blob/199de5f3a493a9bf2dcc7fa5bf841d7759c13d7d/LICENSE:
+--------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+--------------------------------------------------------------------
+
+
+Dependency: go.uber.org/zap
+Version: v1.14.0
+License type: MIT
+https://github.com/uber-go/zap/blob/0bd02a6308c1bac3a03b02dc385555297cb22f83/LICENSE.txt:
+--------------------------------------------------------------------
+Copyright (c) 2016-2017 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+--------------------------------------------------------------------
+
+
+Dependency: github.com/magefile/mage
+Version: v1.9.0
+License type: Apache-2.0
+https://github.com/magefile/mage/blob/324c6690ed410efc1d9b597e477c46d42cbeb340/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
+--------------------------------------------------------------------
+Dependency: github.com/pkg/errors
+Version: v0.9.1
+License type (autodetected): BSD-2-Clause
+https://github.com/pkg/errors/blob/614d223910a179a466c1767a985424175c39b465/LICENSE
+--------------------------------------------------------------------
+Copyright (c) 2015, Dave Cheney <dave@cheney.net>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+--------------------------------------------------------------------
 --------------------------------------------------------------------
 Dependency: go.elastic.co/fastjson
 Version: v1.0.0
@@ -7539,7 +7650,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: go.uber.org/atomic
-Version: v1.3.1
+Version: v1.5.0
 License type (autodetected): MIT
 ./vendor/go.uber.org/atomic/LICENSE.txt:
 --------------------------------------------------------------------
@@ -7565,8 +7676,7 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: go.uber.org/multierr
-Version: v1.1.1
-Revision: fb7d312c2c04
+Version: v1.3.0
 License type (autodetected): MIT
 ./vendor/go.uber.org/multierr/LICENSE.txt:
 --------------------------------------------------------------------
@@ -7590,9 +7700,35 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
+--------------------------------------------------------------------
+Dependency: go.uber.org/tools
+Revision: 2cfd321de3ee
+License type (autodetected): MIT
+./vendor/go.uber.org/tools/LICENSE:
+--------------------------------------------------------------------
+Copyright (c) 2017 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
 --------------------------------------------------------------------
 Dependency: go.uber.org/zap
-Version: v1.7.1
+Version: v1.14.0
 License type (autodetected): MIT
 ./vendor/go.uber.org/zap/LICENSE.txt:
 --------------------------------------------------------------------

auditbeat/auditbeat.reference.yml

@@ -1294,6 +1294,11 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
+# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
+# information. Recommended to use in combination with `logging.json=true`
+# Defaults to false.
+#logging.ecs: false
+
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.reference.yml

@@ -2007,6 +2007,11 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
+# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
+# information. Recommended to use in combination with `logging.json=true`
+# Defaults to false.
+#logging.ecs: false
+
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

go.mod

@@ -93,7 +93,7 @@ require (
 	github.com/gorilla/mux v1.7.2 // indirect
 	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect
-	github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874
+	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d // indirect
 	github.com/insomniacslk/dhcp v0.0.0-20180716145214-633285ba52b2
 	github.com/jcmturner/gofork v1.0.0 // indirect
@@ -135,7 +135,7 @@ require (
 	github.com/shirou/gopsutil v2.19.11+incompatible
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.0
 	github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
 	github.com/tsg/gopacket v0.0.0-20190320122513-dd3d0e41124a
 	github.com/urso/ecslog v0.0.1
@@ -144,9 +144,10 @@ require (
 	go.elastic.co/apm v1.7.2
 	go.elastic.co/apm/module/apmelasticsearch v1.7.2
 	go.elastic.co/apm/module/apmhttp v1.7.2
-	go.uber.org/atomic v1.3.1
-	go.uber.org/multierr v1.1.1-0.20170829224307-fb7d312c2c04
-	go.uber.org/zap v1.7.1
+	go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193
+	go.uber.org/atomic v1.5.0
+	go.uber.org/multierr v1.3.0
+	go.uber.org/zap v1.14.0
 	golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2