diff --git a/NOTICE.txt b/NOTICE.txt index c1e2717582c..3e47f7145e3 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -512,7 +512,7 @@ Contents of "LICENSE": -------------------------------------------------------------------- Dependency: github.com/elastic/beats/v7 Version: v7.0.0 -Revision: 76db4c043865 +Revision: 553c92eeff2d License type (autodetected): Apache-2.0 -------------------------------------------------------------------- @@ -2562,6 +2562,114 @@ Dependency: go.elastic.co/apm/module/apmhttp Version: v1.7.2 License type (autodetected): Apache-2.0 +-------------------------------------------------------------------- +Dependency: go.elastic.co/ecszap +Version: v0.2.0 +License type (autodetected): Apache-2.0 +Contents of "NOTICE.txt": + + ecszap + Copyright 2020-2020 Elasticsearch B.V. + + ========================================================================== + Third party libraries used by the Elastic ecszap project: + ========================================================================== + + Dependency: github.com/stretchr/testify + Version: v1.4.0 + License type: MIT + https://github.com/stretchr/testify/blob/199de5f3a493a9bf2dcc7fa5bf841d7759c13d7d/LICENSE: + -------------------------------------------------------------------- + MIT License + + Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + -------------------------------------------------------------------- + + + Dependency: go.uber.org/zap + Version: v1.14.0 + License type: MIT + https://github.com/uber-go/zap/blob/0bd02a6308c1bac3a03b02dc385555297cb22f83/LICENSE.txt: + -------------------------------------------------------------------- + Copyright (c) 2016-2017 Uber Technologies, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + -------------------------------------------------------------------- + + + Dependency: github.com/magefile/mage + Version: v1.9.0 + License type: Apache-2.0 + https://github.com/magefile/mage/blob/324c6690ed410efc1d9b597e477c46d42cbeb340/LICENSE: + -------------------------------------------------------------------- + Apache License 2.0 + + + -------------------------------------------------------------------- + Dependency: github.com/pkg/errors + Version: v0.9.1 + License type (autodetected): BSD-2-Clause + https://github.com/pkg/errors/blob/614d223910a179a466c1767a985424175c39b465/LICENSE + -------------------------------------------------------------------- + Copyright (c) 2015, Dave Cheney + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + -------------------------------------------------------------------- + -------------------------------------------------------------------- Dependency: go.elastic.co/fastjson Version: v1.0.0 diff --git a/_meta/beat.yml b/_meta/beat.yml index 7c9364dc454..6811f8dbc1c 100644 --- a/_meta/beat.yml +++ b/_meta/beat.yml @@ -1009,6 +1009,12 @@ output.elasticsearch: # Set to true to log messages in json format. #logging.json: false +# Set to true to log with minimal Elastic Common Schema (ECS) fields set. +# It is recommended to set `logging.json=true` when enabling ECS logging. +# Defaults to false. +#logging.ecs: false + + #=============================== HTTP Endpoint =============================== # apm-server can expose internal metrics through a HTTP endpoint. For security diff --git a/apm-server.docker.yml b/apm-server.docker.yml index 8e89367ae1c..4f7885625e8 100644 --- a/apm-server.docker.yml +++ b/apm-server.docker.yml @@ -1009,6 +1009,12 @@ output.elasticsearch: # Set to true to log messages in json format. #logging.json: false +# Set to true to log with minimal Elastic Common Schema (ECS) fields set. +# It is recommended to set `logging.json=true` when enabling ECS logging. +# Defaults to false. +#logging.ecs: false + + #=============================== HTTP Endpoint =============================== # apm-server can expose internal metrics through a HTTP endpoint. For security diff --git a/apm-server.yml b/apm-server.yml index ca66b8c32af..bac435f0420 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -1009,6 +1009,12 @@ output.elasticsearch: # Set to true to log messages in json format. #logging.json: false +# Set to true to log with minimal Elastic Common Schema (ECS) fields set. +# It is recommended to set `logging.json=true` when enabling ECS logging. +# Defaults to false. +#logging.ecs: false + + #=============================== HTTP Endpoint =============================== # apm-server can expose internal metrics through a HTTP endpoint. For security diff --git a/docs/copied-from-beats/docs/loggingconfig.asciidoc b/docs/copied-from-beats/docs/loggingconfig.asciidoc index 633ca7ca2e4..20f0113d842 100644 --- a/docs/copied-from-beats/docs/loggingconfig.asciidoc +++ b/docs/copied-from-beats/docs/loggingconfig.asciidoc @@ -235,6 +235,12 @@ true. When true, logs messages in JSON format. The default is false. +[float] +==== `logging.ecs` + +When true, logs messages with the minimum required {ecs-ref}/ecs-reference.html[Elastic Common Schema (ECS)] +information. + ifndef::serverless[] [float] ==== `logging.files.redirect_stderr` experimental[] diff --git a/go.mod b/go.mod index 11547c7ffd4..029cb14af52 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/dop251/goja v0.0.0-20200414142002-77e84ffb8c65 // indirect github.com/dop251/goja_nodejs v0.0.0-20200128125109-2d688c7e0ac4 // indirect github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200505055820-76db4c043865 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200505202147-553c92eeff2d github.com/elastic/go-elasticsearch/v7 v7.5.0 github.com/elastic/go-elasticsearch/v8 v8.0.0-20200210103600-aff00e5adfde github.com/elastic/go-hdrhistogram v0.1.0 @@ -36,20 +36,20 @@ require ( github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/procfs v0.0.11 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/reviewdog/reviewdog v0.9.17 github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.0 github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c github.com/uber/tchannel-go v1.16.0 // indirect go.elastic.co/apm v1.7.2 go.elastic.co/apm/module/apmelasticsearch v1.7.2 go.elastic.co/apm/module/apmgrpc v1.7.0 go.elastic.co/apm/module/apmhttp v1.7.2 + go.elastic.co/ecszap v0.2.0 // indirect go.uber.org/atomic v1.6.0 go.uber.org/zap v1.15.0 golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 // indirect diff --git a/go.sum b/go.sum index 76c57701c9f..b73eeb265c5 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= +github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -234,6 +235,8 @@ github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200325221042-5d550fbc2313 h1:WN6UY github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200325221042-5d550fbc2313/go.mod h1:2QDEcZki5w1sRSjfytFDr5bhyeMWAYTHmTzuuZ9U/vM= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200505055820-76db4c043865 h1:tF7bZHaDRXIxKA77ZSYpvzYZTGHbtKVcseDE/LXKT+Q= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200505055820-76db4c043865/go.mod h1:uF1RU0IH/rU4zAc13vd20qbHHR/I0BDSdwsuEBEEy/Y= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200505202147-553c92eeff2d h1:5CASYeefJ5lVGwKyVK8JqLBTu5zwbgI9neov3cJ4FIo= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20200505202147-553c92eeff2d/go.mod h1:StOavAVPJWXqrPg+0srQFWtow+9T756j/ck/lYcwTyM= github.com/elastic/ecs v1.5.0 h1:/VEIBsRU4ecq2+U3RPfKNc6bFyomP6qnthYEcQZu8GU= github.com/elastic/ecs v1.5.0/go.mod h1:pgiLbQsijLOJvFR8OTILLu0Ni/R/foUNg0L+T6mU9b4= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng= @@ -412,6 +415,7 @@ github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -901,6 +905,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.0 h1:DMOzIV76tmoDNE9pX6RSN0aDtCYeCg5VueieJaAo1uw= +github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -984,6 +990,10 @@ go.elastic.co/apm/module/apmhttp v1.7.0 h1:dwUkUHlGR6W7FSAxdsZvO3tz+IaLxlXSnwH7A go.elastic.co/apm/module/apmhttp v1.7.0/go.mod h1:70/fYU6lgIII213g7As10lm2Ca/ZkGixeJBoyfrGKes= go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn/+i34w= go.elastic.co/apm/module/apmhttp v1.7.2/go.mod h1:sTFWiWejnhSdZv6+dMgxGec2Nxe/ZKfHfz/xtRM+cRY= +go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 h1:NjYJ/beChqugXSavTkH5tF6shvr/is8jdgJ331wfwT8= +go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= +go.elastic.co/ecszap v0.2.0 h1:BSZNJ2MOIsecJ7L4ezUA+JIarx14wclqZLJm/mBj044= +go.elastic.co/ecszap v0.2.0/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -998,11 +1008,13 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.1/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.1-0.20170829224307-fb7d312c2c04 h1:8sYuFs2lovgFwQi15/wIkCkGX9sL8RouzbWUmBjTcXk= go.uber.org/multierr v1.1.1-0.20170829224307-fb7d312c2c04/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= @@ -1010,6 +1022,7 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= @@ -1023,6 +1036,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/magefile.go b/magefile.go index 26cf972d608..d94b0e0d78c 100644 --- a/magefile.go +++ b/magefile.go @@ -103,8 +103,8 @@ func Config() error { func shortConfigFileParams() mage.ConfigFileParams { return mage.ConfigFileParams{ - ShortParts: []string{ - mage.OSSBeatDir("_meta/beat.yml"), + Short: mage.ConfigParams{ + Template: mage.OSSBeatDir("_meta/beat.yml"), }, ExtraVars: map[string]interface{}{ "elasticsearch_hostport": "localhost:9200", @@ -117,8 +117,8 @@ func shortConfigFileParams() mage.ConfigFileParams { func dockerConfigFileParams() mage.ConfigFileParams { return mage.ConfigFileParams{ - DockerParts: []string{ - mage.OSSBeatDir("_meta/beat.yml"), + Docker: mage.ConfigParams{ + Template: mage.OSSBeatDir("_meta/beat.yml"), }, ExtraVars: map[string]interface{}{ "elasticsearch_hostport": "elasticsearch:9200", diff --git a/testing/environments/docker/elasticsearch/kerberos/init.sh b/testing/environments/docker/elasticsearch/kerberos/init.sh new file mode 100644 index 00000000000..ac7fe70fa69 --- /dev/null +++ b/testing/environments/docker/elasticsearch/kerberos/init.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# setup Keberos +echo elasticsearch_kerberos.elastic > /etc/hostname && echo "127.0.0.1 elasticsearch_kerberos.elastic" >> /etc/hosts + +/scripts/installkdc.sh +/scripts/addprincs.sh + +# add test user +bin/elasticsearch-users useradd beats -r superuser -p testing | /usr/local/bin/docker-entrypoint.sh eswrapper diff --git a/testing/environments/docker/elasticsearch/kerberos/installkdc.sh b/testing/environments/docker/elasticsearch/kerberos/installkdc.sh new file mode 100644 index 00000000000..f35848d004c --- /dev/null +++ b/testing/environments/docker/elasticsearch/kerberos/installkdc.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# KDC installation steps and considerations based on https://web.mit.edu/kerberos/krb5-latest/doc/admin/install_kdc.html +# and helpful input from https://help.ubuntu.com/community/Kerberos + +LOCALSTATEDIR=/etc +LOGDIR=/var/log/krb5 + +#MARKER_FILE=/etc/marker + +# Transfer and interpolate krb5.conf +cp /config/krb5.conf.template $LOCALSTATEDIR/krb5.conf +sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5.conf +sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5.conf +sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5.conf +sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf + + +# Transfer and interpolate the kdc.conf +mkdir -p $LOCALSTATEDIR/krb5kdc +cp /config/kdc.conf.template $LOCALSTATEDIR/krb5kdc/kdc.conf +sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf +sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf +sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf +sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf + +# Touch logging locations +mkdir -p $LOGDIR +touch $LOGDIR/kadmin.log +touch $LOGDIR/krb5kdc.log +touch $LOGDIR/krb5lib.log + +# Update package manager +yum update -qqy + +# Install krb5 packages +yum install -qqy krb5-{server,libs,workstation} + +# Create kerberos database with stash file and garbage password +kdb5_util create -s -r $REALM_NAME -P zyxwvutsrpqonmlk9876 + +# Set up admin acls +cat << EOF > /etc/krb5kdc/kadm5.acl +*/admin@$REALM_NAME * +*@$REALM_NAME * +*/*@$REALM_NAME i +EOF + +# Create admin principal +kadmin.local -q "addprinc -pw elastic admin/admin@$REALM_NAME" +kadmin.local -q "ktadd -k /etc/admin.keytab admin/admin@$REALM_NAME" + +# Create a link so addprinc.sh is on path +ln -s /scripts/addprinc.sh /usr/bin/ diff --git a/testing/environments/docker/elasticsearch_kerberos/Dockerfile b/testing/environments/docker/elasticsearch_kerberos/Dockerfile new file mode 100644 index 00000000000..49b29dfb51d --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/Dockerfile @@ -0,0 +1,15 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0-SNAPSHOT + +ADD scripts /scripts +ADD config /config +ADD healthcheck.sh /healthcheck.sh +ADD start.sh /start.sh + +ENV REALM_NAME ELASTIC +ENV KDC_NAME elasticsearch_kerberos.elastic +ENV BUILD_ZONE elastic +ENV ELASTIC_ZONE $BUILD_ZONE + +USER root +RUN /scripts/installkdc.sh && /scripts/addprincs.sh +USER elasticsearch diff --git a/testing/environments/docker/elasticsearch_kerberos/config/kdc.conf.template b/testing/environments/docker/elasticsearch_kerberos/config/kdc.conf.template new file mode 100644 index 00000000000..0d32b8d411f --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/config/kdc.conf.template @@ -0,0 +1,34 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[kdcdefaults] + kdc_listen = 1088 + kdc_tcp_listen = 1088 + +[realms] + ${REALM_NAME} = { + kadmind_port = 1749 + max_life = 12h 0m 0s + max_renewable_life = 7d 0h 0m 0s + master_key_type = aes256-cts + supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal + } + +[logging] + kdc = FILE:/var/log/krb5/krb5kdc.log + admin_server = FILE:/var/log/krb5/kadmin.log + default = FILE:/var/log/krb5/krb5lib.log diff --git a/testing/environments/docker/elasticsearch_kerberos/config/krb5.conf b/testing/environments/docker/elasticsearch_kerberos/config/krb5.conf new file mode 100644 index 00000000000..1b34299558c --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/config/krb5.conf @@ -0,0 +1,25 @@ +[libdefaults] + default_realm = ELASTIC + dns_canonicalize_hostname = false + dns_lookup_kdc = false + dns_lookup_realm = false + dns_uri_lookup = false + forwardable = true + ignore_acceptor_hostname = true + rdns = false + default_tgs_enctypes = aes128-cts-hmac-sha1-96 + default_tkt_enctypes = aes128-cts-hmac-sha1-96 + permitted_enctypes = aes128-cts-hmac-sha1-96 + kdc_timeout = 3000 + +[realms] + ELASTIC = { + kdc = elasticsearch_kerberos.elastic:88 + admin_server = elasticsearch_kerberos.elastic:749 + default_domain = elastic + } + +[domain_realm] + .elastic = ELASTIC + elastic = ELASTIC + diff --git a/testing/environments/docker/elasticsearch_kerberos/config/krb5.conf.template b/testing/environments/docker/elasticsearch_kerberos/config/krb5.conf.template new file mode 100644 index 00000000000..75245ab7733 --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/config/krb5.conf.template @@ -0,0 +1,43 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[libdefaults] + default_realm = ${REALM_NAME} + dns_canonicalize_hostname = false + dns_lookup_kdc = false + dns_lookup_realm = false + dns_uri_lookup = false + forwardable = true + ignore_acceptor_hostname = true + rdns = false + default_tgs_enctypes = aes128-cts-hmac-sha1-96 + default_tkt_enctypes = aes128-cts-hmac-sha1-96 + permitted_enctypes = aes128-cts-hmac-sha1-96 + udp_preference_limit = 1 + kdc_timeout = 3000 + +[realms] + ${REALM_NAME} = { + kdc = localhost:1088 + admin_server = localhost:1749 + default_domain = ${BUILD_ZONE} + } + +[domain_realm] + .${ELASTIC_ZONE} = ${REALM_NAME} + ${ELASTIC_ZONE} = ${REALM_NAME} + diff --git a/testing/environments/docker/elasticsearch_kerberos/healthcheck.sh b/testing/environments/docker/elasticsearch_kerberos/healthcheck.sh new file mode 100644 index 00000000000..a0932afaa94 --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/healthcheck.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# check if service principal is OK +KRB5_CONFIG=/etc/krb5.conf \ + kinit -k -t /etc/HTTP_elasticsearch_kerberos.elastic.keytab HTTP/elasticsearch_kerberos.elastic@ELASTIC + + +# check if beats user can connect +echo testing | KRB5_CONFIG=/etc/krb5.conf kinit beats@ELASTIC +klist +curl --negotiate -u : -XGET http://elasticsearch_kerberos.elastic:9200/ diff --git a/testing/environments/docker/elasticsearch_kerberos/init.sh b/testing/environments/docker/elasticsearch_kerberos/init.sh new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/environments/docker/elasticsearch_kerberos/scripts/addprinc.sh b/testing/environments/docker/elasticsearch_kerberos/scripts/addprinc.sh new file mode 100644 index 00000000000..97493df7c51 --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/scripts/addprinc.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +if [[ $# -lt 1 ]]; then + echo 'Usage: addprinc.sh principalName [password]' + echo ' principalName user principal name without realm' + echo ' password If provided then will set password for user else it will provision user with keytab' + exit 1 +fi + +PRINC="$1" +PASSWD="$2" +USER=$(echo $PRINC | tr "/" "_") +REALM=ELASTIC + +VDIR=/usr/share/kerberos +BUILD_DIR=/var/build +LOCALSTATEDIR=/etc +LOGDIR=/var/log/krb5 + +ADMIN_PRIN=admin/admin@$REALM +ADMIN_KTAB=$LOCALSTATEDIR/admin.keytab + +USER_PRIN=$PRINC@$REALM +USER_KTAB=$LOCALSTATEDIR/$USER.keytab + +if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then + echo "Principal '${PRINC}@${REALM}' already exists. Re-copying keytab..." + sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab +else + if [ -z "$PASSWD" ]; then + echo "Provisioning '${PRINC}@${REALM}' principal and keytab..." + sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN" + sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN" + sudo chmod 777 $USER_KTAB + sudo cp $USER_KTAB /usr/share/elasticsearch/config + sudo chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/$USER.keytab + else + echo "Provisioning '${PRINC}@${REALM}' principal with password..." + sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC" + fi +fi + +echo "Done provisioning $USER" diff --git a/testing/environments/docker/elasticsearch_kerberos/scripts/addprincs.sh b/testing/environments/docker/elasticsearch_kerberos/scripts/addprincs.sh new file mode 100644 index 00000000000..7ee85889f0d --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/scripts/addprincs.sh @@ -0,0 +1,7 @@ +set -e + +krb5kdc +kadmind + +addprinc.sh HTTP/elasticsearch_kerberos.elastic +addprinc.sh beats testing diff --git a/testing/environments/docker/elasticsearch_kerberos/scripts/installkdc.sh b/testing/environments/docker/elasticsearch_kerberos/scripts/installkdc.sh new file mode 100644 index 00000000000..50ab0ff0a6a --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/scripts/installkdc.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +LOCALSTATEDIR=/etc +KDC_CONFIG=/var/kerberos +LOGDIR=/var/log/krb5 + +#MARKER_FILE=/etc/marker + +# Transfer and interpolate krb5.conf +cp /config/krb5.conf.template $LOCALSTATEDIR/krb5.conf +sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5.conf +sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5.conf +sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5.conf +sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf + + +# Transfer and interpolate the kdc.conf +mkdir -p $KDC_CONFIG/krb5kdc +cp /config/kdc.conf.template $KDC_CONFIG/krb5kdc/kdc.conf +sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $KDC_CONFIG/krb5kdc/kdc.conf +sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $KDC_CONFIG/krb5kdc/kdc.conf +sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $KDC_CONFIG/krb5kdc/kdc.conf +sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf + +# Touch logging locations +mkdir -p $LOGDIR +touch $LOGDIR/kadmin.log +touch $LOGDIR/krb5kdc.log +touch $LOGDIR/krb5lib.log + +# Update package manager +yum update -qqy + +# Install krb5 packages +yum install -qqy krb5-{server,libs,workstation} sudo + +# Create kerberos database with stash file and garbage password +kdb5_util create -s -r $REALM_NAME -P zyxwvutsrpqonmlk9876 + +# Set up admin acls +cat << EOF > /var/kerberos/krb5kdc/kadm5.acl +*/admin@$REALM_NAME * +*@$REALM_NAME * +*/*@$REALM_NAME i +EOF + +# Create admin principal +kadmin.local -q "addprinc -pw elastic admin/admin@$REALM_NAME" +kadmin.local -q "ktadd -k /etc/admin.keytab admin/admin@$REALM_NAME" + +# set ownership for ES +chown -R elasticsearch:elasticsearch $LOGDIR +chown -R elasticsearch:elasticsearch $KDC_CONFIG +chown -R elasticsearch:elasticsearch $LOCALSTATEDIR/krb5.conf +chown -R elasticsearch:elasticsearch $LOCALSTATEDIR/admin.keytab + + +# Create a link so addprinc.sh is on path +ln -s /scripts/addprinc.sh /usr/bin/ diff --git a/testing/environments/docker/elasticsearch_kerberos/start.sh b/testing/environments/docker/elasticsearch_kerberos/start.sh new file mode 100644 index 00000000000..522f6c20474 --- /dev/null +++ b/testing/environments/docker/elasticsearch_kerberos/start.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# start Kerberos services +krb5kdc +kadmind + +# start ES +/usr/local/bin/docker-entrypoint.sh eswrapper diff --git a/testing/environments/docker/kafka/Dockerfile b/testing/environments/docker/kafka/Dockerfile index 2af8a1dd0b9..24bf0996193 100644 --- a/testing/environments/docker/kafka/Dockerfile +++ b/testing/environments/docker/kafka/Dockerfile @@ -4,7 +4,7 @@ ENV KAFKA_HOME /kafka # The advertised host is kafka. This means it will not work if container is started locally and connected from localhost to it ENV KAFKA_ADVERTISED_HOST kafka ENV KAFKA_LOGS_DIR="/kafka-logs" -ENV KAFKA_VERSION 2.1.1 +ENV KAFKA_VERSION 2.2.2 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true" ENV TERM=linux diff --git a/testing/environments/docker/kerberos_kdc/Dockerfile b/testing/environments/docker/kerberos_kdc/Dockerfile new file mode 100644 index 00000000000..629fbaebcd5 --- /dev/null +++ b/testing/environments/docker/kerberos_kdc/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:14.04 +ADD scripts /scripts + +ENV REALM_NAME ELASTIC +ENV KDC_NAME kerberos_kdc +ENV BUILD_ZONE elastic +ENV ELASTIC_ZONE $BUILD_ZONE + +RUN echo kerberos_kdc.elastic > /etc/hostname && echo "127.0.0.1 kerberos_kdc.elastic" >> /etc/hosts +RUN bash /scripts/installkdc.sh + +EXPOSE 88 +EXPOSE 749 + +CMD sleep infinity diff --git a/testing/environments/local.yml b/testing/environments/local.yml index 592ff2d0ab8..7d588a82987 100644 --- a/testing/environments/local.yml +++ b/testing/environments/local.yml @@ -16,6 +16,7 @@ services: ports: - "127.0.0.1:5044:5044" - "127.0.0.1:5055:5055" + - "127.0.0.1:9600:9600" depends_on: elasticsearch: condition: service_healthy