Merge branch 'master' into remove-central-mgmt

.go-version

@@ -1 +1 @@
-1.15.12
+1.16.4

CHANGELOG-developer.next.asciidoc

@@ -54,6 +54,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Make implementing `Close` required for `reader.Reader` interfaces. {pull}20455[20455]
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
+- Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 
 ==== Bugfixes
 
@@ -113,4 +114,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.9. {pull}24442[24442]
 - Update Go version to 1.15.10. {pull}24606[24606]
 - Update Go version to 1.15.12. {pull}25629[25629]
+- Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
 - Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]

CHANGELOG.next.asciidoc

@@ -33,6 +33,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
 - Remove id_field_data {pull}25239[25239]
 - Removed beats central management {pull}25696[25696], {issue}23908[23908]
+- MacOSX minimum supported version set to 10.14 {issue}24193{24193}
 
 *Auditbeat*
 
@@ -274,6 +275,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
+- Improve inode reuse handling by removing state for removed files more eagerly from the internal state table in the logs inputs. {pull}25756[25756]
 - Mitigate deadlock is aws-s3 input when SQS visibility timeout is exceeded. {issue}25750[25750]
 - Fix httpjson cursor override with empty values by adding `ignore_empty_value` option. {pull}25802[25802]
 
@@ -585,6 +587,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883]
 - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
 - Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422]
+- Improve ES output error insights. {pull}25825[25825]
 
 *Auditbeat*
 
@@ -936,6 +939,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Refactor state_* metricsets to share response from endpoint. {pull}25640[25640]
 - Add server id to zookeeper events. {pull}25550[25550]
 - Add additional network metrics to docker/network {pull}25354[25354]
+- Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
 
 *Packetbeat*
 

Jenkinsfile

@@ -59,12 +59,12 @@ pipeline {
         dir("${BASE_DIR}") {
             // We use a raw checkout to avoid the many extra objects which are brought in
             // with a `git fetch` as would happen if we used the `gitCheckout` step.
-            checkout scm 
+            checkout scm
         }
         stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
         dir("${BASE_DIR}"){
-          // Skip all the stages except docs for PR's with asciidoc and md changes only
-          setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString())
+          // Skip all the stages except docs for PR's with asciidoc, md or deploy k8s templates changes only
+          setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '(.*\\.(asciidoc|md)|deploy/kubernetes/.*-kubernetes\\.yaml)' ], shouldMatchAll: true).toString())
           setEnvVar('GO_MOD_CHANGES', isGitRegionMatch(patterns: [ '^go.mod' ], shouldMatchAll: false).toString())
           setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '^dev-tools/packaging/.*' ], shouldMatchAll: false).toString())
           setEnvVar('GO_VERSION', readFile(".go-version").trim())
@@ -352,9 +352,9 @@ def packagingLinux(Map args = [:]) {
                 'linux/amd64',
                 'linux/386',
                 'linux/arm64',
-                'linux/armv7',
                 // The platforms above are disabled temporarly as crossbuild images are
                 // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
+                //'linux/armv7',
                 //'linux/ppc64le',
                 //'linux/mips64',
                 //'linux/s390x',

NOTICE.txt

@@ -37760,37 +37760,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/poy/eachers
-Version: v0.0.0-20181020210610-23942921fe77
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/poy/eachers@v0.0.0-20181020210610-23942921fe77/LICENSE.md:
-
-The MIT License (MIT)
-
-Copyright (c) 2016 Andrew Poydence
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/prometheus/client_golang
 Version: v1.1.1-0.20190913103102-20428fa0bffc

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \

dev-tools/mage/crossbuild.go

@@ -198,7 +198,9 @@ func crossBuildImage(platform string) (string, error) {
 
 	switch {
 	case strings.HasPrefix(platform, "darwin"):
-		tagSuffix = "darwin"
+		tagSuffix = "darwin-debian10"
+	case strings.HasPrefix(platform, "linux/armv7"):
+		tagSuffix = "armhf"
 	case strings.HasPrefix(platform, "linux/arm"):
 		tagSuffix = "arm"
 		if runtime.GOARCH == "arm64" {

dev-tools/mage/gotest_test.go

@@ -364,7 +364,7 @@ func TestGoTest_Helper_WithWrongPanic(t *testing.T) {
 	t.Run("setup failing go-routine", func(t *testing.T) {
 		go func() {
 			time.Sleep(1 * time.Second)
-			t.Fatal("oops")
+			t.Error("oops")
 		}()
 	})
 

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \

filebeat/input/log/input.go

@@ -231,21 +231,21 @@ func (p *Input) Run() {
 
 	// It is important that a first scan is run before cleanup to make sure all new states are read first
 	if p.config.CleanInactive > 0 || p.config.CleanRemoved {
-		beforeCount := p.states.Count()
-		cleanedStates, pendingClean := p.states.Cleanup()
-		logger.Debugf("input states cleaned up. Before: %d, After: %d, Pending: %d",
-			beforeCount, beforeCount-cleanedStates, pendingClean)
+		p.cleanupStates()
 	}
 
-	// Marking removed files to be cleaned up. Cleanup happens after next scan to make sure all states are updated first
+	// Marking removed files to be cleaned up.
 	if p.config.CleanRemoved {
+		var removed uint
+
 		for _, state := range p.states.GetStates() {
 			stateLogger := loggerWithState(logger, state)
 
 			// os.Stat will return an error in case the file does not exist
 			stat, err := os.Stat(state.Source)
 			if err != nil {
 				if os.IsNotExist(err) {
+					removed++
 					p.removeState(stateLogger, state)
 					stateLogger.Debugf("Remove state for file as file removed: %s", state.Source)
 				} else {
@@ -259,14 +259,27 @@ func (p *Input) Run() {
 					state.Id, state.IdentifierName = p.fileStateIdentifier.GenerateID(state)
 				}
 				if !state.IsEqual(&newState) {
+					removed++
 					p.removeState(stateLogger, state)
 					stateLogger.Debugf("Remove state of file as its identity has changed: %s", state.Source)
 				}
 			}
 		}
+
+		if removed > 0 {
+			logger.Debugf("%v entries marked as removed. Trigger state cleanup.", removed)
+			p.cleanupStates()
+		}
 	}
 }
 
+func (p *Input) cleanupStates() {
+	beforeCount := p.states.Count()
+	cleanedStates, pendingClean := p.states.Cleanup()
+	p.logger.Debugf("input states cleaned up. Before: %d, After: %d, Pending: %d",
+		beforeCount, beforeCount-cleanedStates, pendingClean)
+}
+
 func (p *Input) removeState(logger *logp.Logger, state file.State) {
 	// Only clean up files where state is Finished
 	if !state.Finished {

filebeat/registrar/registrar.go

@@ -175,6 +175,7 @@ func (r *Registrar) Run() {
 			// flush timeout configured. Only update internal state and track pending
 			// updates to be written to registry.
 			r.onEvents(states)
+			r.gcStates()
 			if flushC == nil && len(states) > 0 {
 				timer = time.NewTimer(r.flushTimeout)
 				flushC = timer.C
@@ -197,14 +198,12 @@ func (r *Registrar) commitStateUpdates() {
 	statesCurrent.Set(int64(len(states)))
 
 	registryWrites.Inc()
-
-	r.log.Debugf("Registry file updated. %d active states.", len(states))
-	registrySuccess.Inc()
-
 	if err := writeStates(r.store, states); err != nil {
 		r.log.Errorf("Error writing registrar state to statestore: %v", err)
 		registryFails.Inc()
 	}
+	r.log.Debugf("Registry file updated. %d active states.", len(states))
+	registrySuccess.Inc()
 
 	if r.out != nil {
 		r.out.Published(r.bufferedStateUpdates)

generator/common/beatgen/setup/setup.go

@@ -151,5 +151,9 @@ func GitAdd() error {
 
 // Update updates the generated files (aka make update).
 func Update() error {
-	return sh.Run("make", "update")
+	err := sh.Run("make", "update")
+	if err != nil {
+		return err
+	}
+	return gotool.Mod.Tidy()
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/beats/v7
 
-go 1.15
+go 1.16
 
 require (
 	4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b
@@ -133,7 +133,6 @@ require (
 	github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 // indirect
 	github.com/prometheus/client_golang v1.1.1-0.20190913103102-20428fa0bffc // indirect
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
 	github.com/prometheus/common v0.7.0

go.sum

@@ -676,8 +676,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 h1:SNdqPRvRsVmYR0gKqFvrUKhFizPJ6yDiGQ++VAJIoDg=
-github.com/poy/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:x1vqpbcMW9T/KRcQ4b48diSiSVtYgvwQ5xzDByEg4WE=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=

heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \

journalbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \

journalbeat/magefile.go

@@ -137,6 +137,8 @@ func selectImage(platform string) (string, error) {
 	tagSuffix := "main"
 
 	switch {
+	case strings.HasPrefix(platform, "linux/armv7"):
+		tagSuffix = "armhf"
 	case strings.HasPrefix(platform, "linux/arm"):
 		tagSuffix = "arm"
 		if runtime.GOARCH == "arm64" {

libbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \

libbeat/common/transport/logging.go

@@ -0,0 +1,60 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package transport
+
+import (
+	"io"
+	"net"
+
+	"github.com/elastic/beats/v7/libbeat/logp"
+)
+
+type loggingConn struct {
+	net.Conn
+	logger *logp.Logger
+}
+
+func LoggingDialer(d Dialer, logger *logp.Logger) Dialer {
+	return DialerFunc(func(network, addr string) (net.Conn, error) {
+		logger := logger.With("network", network, "address", addr)
+		c, err := d.Dial(network, addr)
+		if err != nil {
+			logger.Errorf("Error dialing %v", err)
+			return nil, err
+		}
+
+		logger.Debugf("Completed dialing successfully")
+		return &loggingConn{c, logger}, nil
+	})
+}
+
+func (l *loggingConn) Read(b []byte) (int, error) {
+	n, err := l.Conn.Read(b)
+	if err != nil && err != io.EOF {
+		l.logger.Debugf("Error reading from connection: %v", err)
+	}
+	return n, err
+}
+
+func (l *loggingConn) Write(b []byte) (int, error) {
+	n, err := l.Conn.Write(b)
+	if err != nil && err != io.EOF {
+		l.logger.Debugf("Error writing to connection: %v", err)
+	}
+	return n, err
+}

libbeat/common/transport/stats.go

@@ -18,6 +18,8 @@
 package transport
 
 import (
+	"context"
+	"io"
 	"net"
 )
 
@@ -42,7 +44,7 @@ func StatsDialer(d Dialer, s IOStatser) Dialer {
 
 func (s *statsConn) Read(b []byte) (int, error) {
 	n, err := s.Conn.Read(b)
-	if err != nil {
+	if err != nil && err != io.EOF && err != context.Canceled {
 		s.stats.ReadError(err)
 	}
 	s.stats.ReadBytes(n)
@@ -51,7 +53,7 @@ func (s *statsConn) Read(b []byte) (int, error) {
 
 func (s *statsConn) Write(b []byte) (int, error) {
 	n, err := s.Conn.Write(b)
-	if err != nil {
+	if err != nil && err != io.EOF && err != context.Canceled {
 		s.stats.WriteError(err)
 	}
 	s.stats.WriteBytes(n)

libbeat/docs/version.asciidoc

@@ -1,6 +1,6 @@
 :stack-version: 8.0.0
 :doc-branch: master
-:go-version: 1.15.12
+:go-version: 1.16.4
 :release-state: unreleased
 :python: 3.7
 :docker: 1.12