diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index eeb647153405..b577bc5a356e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -65,6 +65,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di - Update init scripts to use the `test config` subcommand instead of the deprecated `-configtest` flag. {issue}4600[4600] - Get by default the credentials for connecting to Kibana from the Elasticsearch output configuration. {pull}4867[4867] +- Added `cloud.id` and `cloud.auth` settings, for simplifying using Beats with the Elastic Cloud. {issue}4959[4959] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index f802d5493639..f3992197294c 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -175,6 +175,19 @@ auditbeat.modules: #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index 35fd3d040eee..fac44ac54719 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -59,7 +59,7 @@ auditbeat.modules: #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -69,6 +69,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index b0c7ace7c0de..c7892146cc60 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -566,6 +566,19 @@ filebeat.prospectors: #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index a49d550f818d..8650eb67824d 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -105,7 +105,7 @@ filebeat.config.modules: #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -115,6 +115,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 8b441970e5e7..f1f78e7d54e8 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -324,6 +324,19 @@ heartbeat.scheduler: #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index 308b81051593..998d94863aff 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -52,7 +52,7 @@ heartbeat.monitors: #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -62,6 +62,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 0557b3be5090..7f45529012b8 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -110,6 +110,19 @@ #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml index 5bc54c106d33..d3847a08aecd 100644 --- a/libbeat/_meta/config.yml +++ b/libbeat/_meta/config.yml @@ -29,7 +29,7 @@ #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -39,6 +39,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 980235421603..d4f7514ec0f1 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -47,6 +47,7 @@ import ( "github.com/elastic/beats/libbeat/api" "github.com/elastic/beats/libbeat/cfgfile" + "github.com/elastic/beats/libbeat/cloudid" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/common/file" @@ -443,6 +444,11 @@ func (b *Beat) configure() error { return fmt.Errorf("error loading config file: %v", err) } + err = cloudid.OverwriteSettings(cfg) + if err != nil { + return err + } + b.RawConfig = cfg err = cfg.Unpack(&b.Config) if err != nil { diff --git a/libbeat/cloudid/cloudid.go b/libbeat/cloudid/cloudid.go new file mode 100644 index 000000000000..fc5b3b6a2dfb --- /dev/null +++ b/libbeat/cloudid/cloudid.go @@ -0,0 +1,128 @@ +// package cloudid contains functions for parsing the cloud.id and cloud.auth +// settings and modifying the configuration to take them into account. +package cloudid + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +// OverwriteSettings modifies the received config object by overwriting the +// output.elasticsearch.hosts, output.elasticsearch.username, output.elasticsearch.password, +// setup.kibana.host settings based on values derived from the cloud.id and cloud.auth +// settings. +func OverwriteSettings(cfg *common.Config) error { + + cloudID, _ := cfg.String("cloud.id", -1) + cloudAuth, _ := cfg.String("cloud.auth", -1) + + if cloudID == "" && cloudAuth == "" { + // nothing to hack + return nil + } + + logp.Debug("cloudid", "cloud.id: %s, cloud.auth: %s", cloudID, cloudAuth) + if cloudID == "" { + return errors.New("cloud.auth specified but cloud.id is empty. Please specify both.") + } + + // cloudID overwrites + esURL, kibanaURL, err := decodeCloudID(cloudID) + if err != nil { + return errors.Errorf("Error decoding cloud.id: %v", err) + } + + logp.Info("Setting Elasticsearch and Kibana URLs based on the cloud id: output.elasticsearch.hosts=%s and setup.kibana.host=%s", esURL, kibanaURL) + + esURLConfig, err := common.NewConfigFrom([]string{esURL}) + if err != nil { + return err + } + + // Before enabling the ES output, check that no other output is enabled + tmp := struct { + Output common.ConfigNamespace `config:"output"` + }{} + if err := cfg.Unpack(&tmp); err != nil { + return err + } + if out := tmp.Output; out.IsSet() && out.Name() != "elasticsearch" { + return errors.Errorf("The cloud.id setting enables the Elasticsearch output, but you already have the %s output enabled in the config", out.Name()) + } + + err = cfg.SetChild("output.elasticsearch.hosts", -1, esURLConfig) + if err != nil { + return err + } + + err = cfg.SetString("setup.kibana.host", -1, kibanaURL) + if err != nil { + return err + } + + if cloudAuth != "" { + // cloudAuth overwrites + username, password, err := decodeCloudAuth(cloudAuth) + if err != nil { + return err + } + + err = cfg.SetString("output.elasticsearch.username", -1, username) + if err != nil { + return err + } + + err = cfg.SetString("output.elasticsearch.password", -1, password) + if err != nil { + return err + } + } + + return nil +} + +// decodeCloudID decodes the cloud.id into elasticsearch-URL and kibana-URL +func decodeCloudID(cloudID string) (string, string, error) { + + // 1. Ignore anything before `:`. + idx := strings.LastIndex(cloudID, ":") + if idx >= 0 { + cloudID = cloudID[idx+1:] + } + + // 2. base64 decode + decoded, err := base64.StdEncoding.DecodeString(cloudID) + if err != nil { + return "", "", errors.Wrapf(err, "base64 decoding failed on %s", cloudID) + } + + // 3. separate based on `$` + words := strings.Split(string(decoded), "$") + if len(words) < 3 { + return "", "", errors.Errorf("Expected at least 3 parts in %s", string(decoded)) + } + + // 4. form the URLs + esURL := url.URL{Scheme: "https", Host: fmt.Sprintf("%s.%s:443", words[1], words[0])} + kibanaURL := url.URL{Scheme: "https", Host: fmt.Sprintf("%s.%s:443", words[2], words[0])} + + return esURL.String(), kibanaURL.String(), nil +} + +// decodeCloudAuth splits the cloud.auth into username and password. +func decodeCloudAuth(cloudAuth string) (string, string, error) { + + idx := strings.Index(cloudAuth, ":") + if idx < 0 { + return "", "", errors.New("cloud.auth setting doesn't contain `:` to split between username and password") + } + + return cloudAuth[0:idx], cloudAuth[idx+1:], nil +} diff --git a/libbeat/cloudid/cloudid_test.go b/libbeat/cloudid/cloudid_test.go new file mode 100644 index 000000000000..68ef48e78de1 --- /dev/null +++ b/libbeat/cloudid/cloudid_test.go @@ -0,0 +1,209 @@ +package cloudid + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestDecode(t *testing.T) { + tests := []struct { + cloudID string + expectedEsURL string + expectedKibanaURL string + }{ + { + cloudID: "staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==", + expectedEsURL: "https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443", + expectedKibanaURL: "https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443", + }, + { + cloudID: "dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==", + expectedEsURL: "https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443", + expectedKibanaURL: "https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443", + }, + { + cloudID: ":dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==", + expectedEsURL: "https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443", + expectedKibanaURL: "https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443", + }, + { + cloudID: "gcp-cluster:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZmNlJDBjZDVjZDU2OGVlYmU1M2M4OWViN2NhZTViYWM4YjM3", + expectedEsURL: "https://8a0283af041f195f7729bc04c66a0fce.us-central1.gcp.cloud.es.io:443", + expectedKibanaURL: "https://0cd5cd568eebe53c89eb7cae5bac8b37.us-central1.gcp.cloud.es.io:443", + }, + } + + for _, test := range tests { + esURL, kbURL, err := decodeCloudID(test.cloudID) + assert.NoError(t, err, test.cloudID) + + assert.Equal(t, esURL, test.expectedEsURL, test.cloudID) + assert.Equal(t, kbURL, test.expectedKibanaURL, test.cloudID) + } +} + +func TestDecodeError(t *testing.T) { + tests := []struct { + cloudID string + errorMsg string + }{ + { + cloudID: "staging:garbagedXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==", + errorMsg: "base64 decoding failed", + }, + { + cloudID: "dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZg==", + errorMsg: "Expected at least 3 parts", + }, + } + + for _, test := range tests { + _, _, err := decodeCloudID(test.cloudID) + assert.Error(t, err, test.cloudID) + assert.Contains(t, err.Error(), test.errorMsg, test.cloudID) + } +} + +func TestOverwriteSettings(t *testing.T) { + tests := []struct { + name string + inCfg map[string]interface{} + outCfg map[string]interface{} + }{ + { + name: "No cloud-id specified, nothing should change", + inCfg: map[string]interface{}{ + "output.elasticsearch.hosts": "localhost:9200", + }, + outCfg: map[string]interface{}{ + "output.elasticsearch.hosts": "localhost:9200", + }, + }, + { + name: "cloudid realistic example", + inCfg: map[string]interface{}{ + "output.elasticsearch.hosts": "localhost:9200", + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + "cloud.auth": "elastic:changeme", + }, + outCfg: map[string]interface{}{ + "output.elasticsearch.hosts": []interface{}{"https://249f3af1f4eee24a84e3b401e68a1b2a.us-east-1.aws.found.io:443"}, + "output.elasticsearch.username": "elastic", + "output.elasticsearch.password": "changeme", + "setup.kibana.host": "https://d4ac7559d4674b7c91abe10856d84304.us-east-1.aws.found.io:443", + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + "cloud.auth": "elastic:changeme", + }, + }, + { + name: "only cloudid specified", + inCfg: map[string]interface{}{ + "output.elasticsearch.hosts": "localhost:9200", + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + }, + outCfg: map[string]interface{}{ + "output.elasticsearch.hosts": []interface{}{"https://249f3af1f4eee24a84e3b401e68a1b2a.us-east-1.aws.found.io:443"}, + "setup.kibana.host": "https://d4ac7559d4674b7c91abe10856d84304.us-east-1.aws.found.io:443", + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + }, + }, + { + name: "no output defined", + inCfg: map[string]interface{}{ + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + }, + outCfg: map[string]interface{}{ + "output.elasticsearch.hosts": []interface{}{"https://249f3af1f4eee24a84e3b401e68a1b2a.us-east-1.aws.found.io:443"}, + "setup.kibana.host": "https://d4ac7559d4674b7c91abe10856d84304.us-east-1.aws.found.io:443", + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + }, + }, + { + name: "multiple hosts to overwrite", + inCfg: map[string]interface{}{ + "output.elasticsearch.hosts": []string{"localhost:9200", "test", "test1"}, + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + }, + outCfg: map[string]interface{}{ + "output.elasticsearch.hosts": []interface{}{"https://249f3af1f4eee24a84e3b401e68a1b2a.us-east-1.aws.found.io:443"}, + "setup.kibana.host": "https://d4ac7559d4674b7c91abe10856d84304.us-east-1.aws.found.io:443", + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + }, + }, + } + + for _, test := range tests { + t.Logf("Executing test: %s", test.name) + + cfg, err := common.NewConfigFrom(test.inCfg) + assert.NoError(t, err) + + err = OverwriteSettings(cfg) + assert.NoError(t, err) + + var res map[string]interface{} + err = cfg.Unpack(&res) + assert.NoError(t, err) + + var expected map[string]interface{} + expectedCfg, err := common.NewConfigFrom(test.outCfg) + assert.NoError(t, err) + err = expectedCfg.Unpack(&expected) + assert.NoError(t, err) + + assert.Equal(t, res, expected) + } +} + +func TestOverwriteErrors(t *testing.T) { + tests := []struct { + name string + inCfg map[string]interface{} + errMsg string + }{ + { + name: "cloud.auth specified but cloud.id not", + inCfg: map[string]interface{}{ + "cloud.auth": "elastic:changeme", + }, + errMsg: "cloud.auth specified but cloud.id is empty", + }, + { + name: "invalid cloud.id", + inCfg: map[string]interface{}{ + "cloud.id": "blah", + }, + errMsg: "Error decoding cloud.id", + }, + { + name: "invalid cloud.auth", + inCfg: map[string]interface{}{ + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + "cloud.auth": "blah", + }, + errMsg: "cloud.auth setting doesn't contain `:`", + }, + { + name: "logstash output enabled", + inCfg: map[string]interface{}{ + "cloud.id": "cloudidtest:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQyNDlmM2FmMWY0ZWVlMjRhODRlM2I0MDFlNjhhMWIyYSRkNGFjNzU1OWQ0Njc0YjdjOTFhYmUxMDg1NmQ4NDMwNA==", + "output.logstash.hosts": "localhost:544", + }, + errMsg: "The cloud.id setting enables the Elasticsearch output, but you already have the logstash output enabled", + }, + } + + for _, test := range tests { + t.Logf("Executing test: %s", test.name) + + cfg, err := common.NewConfigFrom(test.inCfg) + assert.NoError(t, err) + + err = OverwriteSettings(cfg) + assert.Error(t, err) + assert.Contains(t, err.Error(), test.errMsg) + } +} diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 751dd55650a8..dcefc0ce6e7c 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -1103,3 +1103,39 @@ output.console: string: '%{[@timestamp]} %{[message]}' ------------------------------------------------------------------------------ +[[configure-cloud-id]] +=== Configure the output for the Elastic Cloud + +{beatname_uc} comes with two settings that simplify the output configuration +when used together with https://cloud.elastic.co/[Elastic Cloud]. When defined, +these setting overwrite settings from other parts in the configuration. + +Example: + +[source,yaml] +------------------------------------------------------------------------------ +cloud.id: "staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==" +cloud.auth: "elastic:changeme" +------------------------------------------------------------------------------ + +These settings can be also specified at the command line, like this: + + +["source","sh",subs="attributes,callouts"] +------------------------------------------------------------------------------ +{beatname_lc} -e -E cloud.id="" -E cloud.auth="" +------------------------------------------------------------------------------ + + +==== `cloud.id` + +The Cloud ID, which can be found in the Elastic Cloud web console, is used by +{beatname_uc} to resolve the Elasticsearch and Kibana URLs. This setting +overwrites the `output.elasticsearch.hosts` and `setup.kibana.host` settings. + +==== `cloud.auth` + +When specified, the `cloud.auth` overwrites the `output.elasticsearch.username` and +`output.elasticsearch.password` settings. Because the Kibana settings inherit +the username and password from the Elasticsearch output, this can also be used +to set the `setup.kibana.username` and `setup.kibana.password` options. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4af153c3f40f..21e825bc1482 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -530,6 +530,19 @@ metricbeat.modules: #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 632432709983..dcca623d8208 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -54,7 +54,7 @@ setup.template.settings: #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -64,6 +64,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 2bec7ccb6ba4..2f87a7e1cd1d 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -562,6 +562,19 @@ packetbeat.protocols: #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 9e5b1c3e61d6..a4ce7c78b2d2 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -123,7 +123,7 @@ packetbeat.protocols: #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -133,6 +133,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index d2407689e328..b2594e0f6a8b 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -139,6 +139,19 @@ winlogbeat.event_logs: #processors: #- add_docker_metadata: ~ +#============================= Elastic Cloud ================================== + +# These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index ed162e150994..c3d495601e60 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -53,7 +53,7 @@ winlogbeat.event_logs: #============================== Kibana ===================================== -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. setup.kibana: @@ -63,6 +63,19 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" +#============================= Elastic Cloud ================================== + +# These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat.