add_cloud_metadata asynchronous initialization (elastic#8845)

Now that the add_cloud_metadata is enabled by default in all beats, we
are losing 3 precious seconds every time a Beat is started outside a
supported cloud environment.

This patch makes the cloud detection an asynchronous task so the Beat
can start and only block if initialization is not completed at the time
of the first enrichment.

Running in debug mode bypasses this parallelism as the processor needs
to be initialized when its String() method is called.

CHANGELOG.asciidoc

@@ -150,6 +150,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Allow Bus to buffer events in case listeners are not configured. {pull}8527[8527]
 - Enable `host` and `cloud` metadata processors by default. {pull}8596[8596]
 - Dissect will now flag event on parsing error. {pull}8751[8751]
+- add_cloud_metadata initialization is performed asynchronously to avoid delays on startup. {pull}8845[8845]
 
 *Auditbeat*
 

libbeat/processors/add_cloud_metadata/add_cloud_metadata.go

@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -314,34 +315,55 @@ func newCloudMetadata(c *common.Config) (processors.Processor, error) {
 		return nil, err
 	}
 
-	result := fetchMetadata(fetchers, config.Timeout)
-	if result == nil {
-		logp.Info("add_cloud_metadata: hosting provider type not detected.")
-		return &addCloudMetadata{}, nil
+	p := &addCloudMetadata{
+		initData: &initData{fetchers, config.Timeout},
 	}
 
-	logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v",
-		result.provider, result.metadata.String())
+	go p.initOnce.Do(p.init)
+	return p, nil
+}
 
-	return &addCloudMetadata{metadata: result.metadata}, nil
+type initData struct {
+	fetchers []*metadataFetcher
+	timeout  time.Duration
 }
 
 type addCloudMetadata struct {
+	initOnce sync.Once
+	initData *initData
 	metadata common.MapStr
 }
 
-func (p addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) {
-	if len(p.metadata) == 0 {
+func (p *addCloudMetadata) init() {
+	result := fetchMetadata(p.initData.fetchers, p.initData.timeout)
+	if result == nil {
+		logp.Info("add_cloud_metadata: hosting provider type not detected.")
+		return
+	}
+	p.metadata = result.metadata
+	p.initData = nil
+	logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v",
+		result.provider, result.metadata.String())
+}
+
+func (p *addCloudMetadata) getMeta() common.MapStr {
+	p.initOnce.Do(p.init)
+	return p.metadata
+}
+
+func (p *addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) {
+	meta := p.getMeta()
+	if len(meta) == 0 {
 		return event, nil
 	}
 
 	// This overwrites the meta.cloud if it exists. But the cloud key should be
 	// reserved for this processor so this should happen.
-	_, err := event.PutValue("meta.cloud", p.metadata)
+	_, err := event.PutValue("meta.cloud", meta)
 
 	return event, err
 }
 
-func (p addCloudMetadata) String() string {
-	return "add_cloud_metadata=" + p.metadata.String()
+func (p *addCloudMetadata) String() string {
+	return "add_cloud_metadata=" + p.getMeta().String()
 }