Cherry-pick #21851 to 7.x: Refactor docker watcher to fix flaky test and other small issues

CHANGELOG.next.asciidoc

@@ -200,6 +200,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
 - The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258]
 - Orderly close processors when processing pipelines are not needed anymore to release their resources. {pull}16349[16349]
+- Fix memory leak and events duplication in docker autodiscover and add_docker_metadata. {pull}21851[21851]
 
 *Auditbeat*
 

libbeat/common/docker/watcher.go

@@ -20,7 +20,8 @@
 package docker
 
 import (
-	"fmt"
+	"context"
+	"io"
 	"net/http"
 	"sync"
 	"time"
@@ -29,7 +30,6 @@ import (
 	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/go-connections/tlsconfig"
-	"golang.org/x/net/context"
 
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 	"github.com/elastic/beats/v7/libbeat/logp"
@@ -39,7 +39,6 @@ import (
 const (
 	shortIDLen                         = 12
 	dockerRequestTimeout               = 10 * time.Second
-	dockerWatchRequestTimeout          = 60 * time.Minute
 	dockerEventsWatchPityTimerInterval = 10 * time.Second
 	dockerEventsWatchPityTimerTimeout  = 10 * time.Minute
 )
@@ -74,20 +73,30 @@ type TLSConfig struct {
 
 type watcher struct {
 	sync.RWMutex
-	log                        *logp.Logger
-	client                     Client
-	ctx                        context.Context
-	stop                       context.CancelFunc
-	containers                 map[string]*Container
-	deleted                    map[string]time.Time // deleted annotations key -> last access time
-	cleanupTimeout             time.Duration
-	lastValidTimestamp         int64
-	lastWatchReceivedEventTime time.Time
-	stopped                    sync.WaitGroup
-	bus                        bus.Bus
-	shortID                    bool // whether to store short ID in "containers" too
+	log            *logp.Logger
+	client         Client
+	ctx            context.Context
+	stop           context.CancelFunc
+	containers     map[string]*Container
+	deleted        map[string]time.Time // deleted annotations key -> last access time
+	cleanupTimeout time.Duration
+	clock          clock
+	stopped        sync.WaitGroup
+	bus            bus.Bus
+	shortID        bool // whether to store short ID in "containers" too
 }
 
+// clock is an interface used to provide mocked time on testing
+type clock interface {
+	Now() time.Time
+}
+
+// systemClock implements the clock interface using the system clock via the time package
+type systemClock struct{}
+
+// Now returns the current time
+func (*systemClock) Now() time.Time { return time.Now() }
+
 // Container info retrieved by the watcher
 type Container struct {
 	ID          string
@@ -147,8 +156,6 @@ func NewWatcher(log *logp.Logger, host string, tls *TLSConfig, storeShortID bool
 
 // NewWatcherWithClient creates a new Watcher from a given Docker client
 func NewWatcherWithClient(log *logp.Logger, client Client, cleanupTimeout time.Duration, storeShortID bool) (Watcher, error) {
-	log = log.Named("docker")
-
 	ctx, cancel := context.WithCancel(context.Background())
 	return &watcher{
 		log:            log,
@@ -160,6 +167,7 @@ func NewWatcherWithClient(log *logp.Logger, client Client, cleanupTimeout time.D
 		cleanupTimeout: cleanupTimeout,
 		bus:            bus.New(log, "docker"),
 		shortID:        storeShortID,
+		clock:          &systemClock{},
 	}, nil
 }
 
@@ -177,7 +185,7 @@ func (w *watcher) Container(ID string) *Container {
 	// Update last access time if it's deleted
 	if ok {
 		w.Lock()
-		w.deleted[container.ID] = time.Now()
+		w.deleted[container.ID] = w.clock.Now()
 		w.Unlock()
 	}
 
@@ -201,7 +209,6 @@ func (w *watcher) Containers() map[string]*Container {
 func (w *watcher) Start() error {
 	// Do initial scan of existing containers
 	w.log.Debug("Start docker containers scanner")
-	w.lastValidTimestamp = time.Now().Unix()
 
 	w.Lock()
 	defer w.Unlock()
@@ -236,108 +243,124 @@ func (w *watcher) Start() error {
 
 func (w *watcher) Stop() {
 	w.stop()
+	w.stopped.Wait()
 }
 
 func (w *watcher) watch() {
-	log := w.log
+	defer w.stopped.Done()
 
 	filter := filters.NewArgs()
 	filter.Add("type", "container")
 
-	for {
+	// Ticker to restart the watcher when no events are received after some time.
+	tickChan := time.NewTicker(dockerEventsWatchPityTimerInterval)
+	defer tickChan.Stop()
+
+	lastValidTimestamp := w.clock.Now()
+
+	watch := func() bool {
+		lastReceivedEventTime := w.clock.Now()
+
+		w.log.Debugf("Fetching events since %s", lastValidTimestamp)
+
 		options := types.EventsOptions{
-			Since:   fmt.Sprintf("%d", w.lastValidTimestamp),
+			Since:   lastValidTimestamp.Format(time.RFC3339Nano),
 			Filters: filter,
 		}
 
-		log.Debugf("Fetching events since %s", options.Since)
-		ctx, cancel := context.WithTimeout(w.ctx, dockerWatchRequestTimeout)
+		ctx, cancel := context.WithCancel(w.ctx)
 		defer cancel()
 
 		events, errors := w.client.Events(ctx, options)
-
-		//ticker for timeout to restart watcher when no events are received
-		w.lastWatchReceivedEventTime = time.Now()
-		tickChan := time.NewTicker(dockerEventsWatchPityTimerInterval)
-		defer tickChan.Stop()
-
-	WATCH:
 		for {
 			select {
 			case event := <-events:
-				log.Debugf("Got a new docker event: %v", event)
-				w.lastValidTimestamp = event.Time
-				w.lastWatchReceivedEventTime = time.Now()
-
-				// Add / update
-				if event.Action == "start" || event.Action == "update" {
-					filter := filters.NewArgs()
-					filter.Add("id", event.Actor.ID)
-
-					containers, err := w.listContainers(types.ContainerListOptions{
-						Filters: filter,
-					})
-					if err != nil || len(containers) != 1 {
-						log.Errorf("Error getting container info: %v", err)
-						continue
-					}
-					container := containers[0]
-
-					w.Lock()
-					w.containers[event.Actor.ID] = container
-					if w.shortID {
-						w.containers[event.Actor.ID[:shortIDLen]] = container
-					}
-					// un-delete if it's flagged (in case of update or recreation)
-					delete(w.deleted, event.Actor.ID)
-					w.Unlock()
-
-					w.bus.Publish(bus.Event{
-						"start":     true,
-						"container": container,
-					})
-				}
-
-				// Delete
-				if event.Action == "die" {
-					container := w.Container(event.Actor.ID)
-					if container != nil {
-						w.bus.Publish(bus.Event{
-							"stop":      true,
-							"container": container,
-						})
-					}
-
-					w.Lock()
-					w.deleted[event.Actor.ID] = time.Now()
-					w.Unlock()
+				w.log.Debugf("Got a new docker event: %v", event)
+				lastValidTimestamp = time.Unix(event.Time, event.TimeNano)
+				lastReceivedEventTime = w.clock.Now()
+
+				switch event.Action {
+				case "start", "update":
+					w.containerUpdate(event)
+				case "die":
+					w.containerDelete(event)
 				}
-
 			case err := <-errors:
-				// Restart watch call
-				if err == context.DeadlineExceeded {
-					log.Info("Context deadline exceeded for docker request, restarting watch call")
-				} else {
-					log.Errorf("Error watching for docker events: %+v", err)
+				switch err {
+				case io.EOF:
+					// Client disconnected, watch is not done, reconnect
+					w.log.Debug("EOF received in events stream, restarting watch call")
+				case context.DeadlineExceeded:
+					w.log.Debug("Context deadline exceeded for docker request, restarting watch call")
+				case context.Canceled:
+					// Parent context has been canceled, watch is done.
+					return true
+				default:
+					w.log.Errorf("Error watching for docker events: %+v", err)
 				}
-
-				time.Sleep(1 * time.Second)
-				break WATCH
-
+				return false
 			case <-tickChan.C:
-				if time.Since(w.lastWatchReceivedEventTime) > dockerEventsWatchPityTimerTimeout {
-					log.Infof("No events received within %s, restarting watch call", dockerEventsWatchPityTimerTimeout)
-					time.Sleep(1 * time.Second)
-					break WATCH
+				if time.Since(lastReceivedEventTime) > dockerEventsWatchPityTimerTimeout {
+					w.log.Infof("No events received within %s, restarting watch call", dockerEventsWatchPityTimerTimeout)
+					return false
 				}
-
 			case <-w.ctx.Done():
-				log.Debug("Watcher stopped")
-				w.stopped.Done()
-				return
+				w.log.Debug("Watcher stopped")
+				return true
 			}
 		}
+	}
 
+	for {
+		done := watch()
+		if done {
+			return
+		}
+		// Wait before trying to reconnect
+		time.Sleep(1 * time.Second)
+	}
+}
+
+func (w *watcher) containerUpdate(event events.Message) {
+	filter := filters.NewArgs()
+	filter.Add("id", event.Actor.ID)
+
+	containers, err := w.listContainers(types.ContainerListOptions{
+		Filters: filter,
+	})
+	if err != nil || len(containers) != 1 {
+		w.log.Errorf("Error getting container info: %v", err)
+		return
+	}
+	container := containers[0]
+
+	w.Lock()
+	w.containers[event.Actor.ID] = container
+	if w.shortID {
+		w.containers[event.Actor.ID[:shortIDLen]] = container
+	}
+	// un-delete if it's flagged (in case of update or recreation)
+	delete(w.deleted, event.Actor.ID)
+	w.Unlock()
+
+	w.bus.Publish(bus.Event{
+		"start":     true,
+		"container": container,
+	})
+}
+
+func (w *watcher) containerDelete(event events.Message) {
+	container := w.Container(event.Actor.ID)
+
+	w.Lock()
+	w.deleted[event.Actor.ID] = w.clock.Now()
+	w.Unlock()
+
+	if container != nil {
+		w.bus.Publish(bus.Event{
+			"stop":      true,
+			"container": container,
+		})
 	}
 }
 
@@ -393,49 +416,52 @@ func (w *watcher) listContainers(options types.ContainerListOptions) ([]*Contain
 
 // Clean up deleted containers after they are not used anymore
 func (w *watcher) cleanupWorker() {
-	log := w.log
+	defer w.stopped.Done()
 
 	for {
 		select {
 		case <-w.ctx.Done():
-			w.stopped.Done()
 			return
 		// Wait a full period
 		case <-time.After(w.cleanupTimeout):
-			// Check entries for timeout
-			var toDelete []string
-			timeout := time.Now().Add(-w.cleanupTimeout)
-			w.RLock()
-			for key, lastSeen := range w.deleted {
-				if lastSeen.Before(timeout) {
-					log.Debugf("Removing container %s after cool down timeout", key)
-					toDelete = append(toDelete, key)
-				}
-			}
-			w.RUnlock()
-
-			// Delete timed out entries:
-			for _, key := range toDelete {
-				container := w.Container(key)
-				if container != nil {
-					w.bus.Publish(bus.Event{
-						"delete":    true,
-						"container": container,
-					})
-				}
-			}
+			w.runCleanup()
+		}
+	}
+}
 
-			w.Lock()
-			for _, key := range toDelete {
-				delete(w.deleted, key)
-				delete(w.containers, key)
-				if w.shortID {
-					delete(w.containers, key[:shortIDLen])
-				}
-			}
-			w.Unlock()
+func (w *watcher) runCleanup() {
+	// Check entries for timeout
+	var toDelete []string
+	timeout := w.clock.Now().Add(-w.cleanupTimeout)
+	w.RLock()
+	for key, lastSeen := range w.deleted {
+		if lastSeen.Before(timeout) {
+			w.log.Debugf("Removing container %s after cool down timeout", key)
+			toDelete = append(toDelete, key)
+		}
+	}
+	w.RUnlock()
+
+	// Delete timed out entries:
+	for _, key := range toDelete {
+		container := w.Container(key)
+		if container != nil {
+			w.bus.Publish(bus.Event{
+				"delete":    true,
+				"container": container,
+			})
+		}
+	}
+
+	w.Lock()
+	for _, key := range toDelete {
+		delete(w.deleted, key)
+		delete(w.containers, key)
+		if w.shortID {
+			delete(w.containers, key[:shortIDLen])
 		}
 	}
+	w.Unlock()
 }
 
 // ListenStart returns a bus listener to receive container started events, with a `container` key holding it