diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0456e3a99e..ee4de95884d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -50,7 +50,7 @@ CHANGELOG* /filebeat/module/santa @elastic/security-external-integrations /filebeat/module/system @elastic/elastic-agent-data-plane /filebeat/module/traefik @elastic/integrations -/heartbeat/ @elastic/uptime +/heartbeat/ @elastic/hosted-services /journalbeat @elastic/elastic-agent-data-plane /libbeat/ @elastic/elastic-agent-data-plane /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs @@ -174,7 +174,7 @@ CHANGELOG* /x-pack/filebeat/module/zscaler @elastic/security-external-integrations /x-pack/filebeat/modules.d/zoom.yml.disabled @elastic/security-external-integrations /x-pack/filebeat/processors/decode_cef/ @elastic/security-external-integrations -/x-pack/heartbeat/ @elastic/uptime +/x-pack/heartbeat/ @elastic/hosted-services /x-pack/metricbeat/ @elastic/elastic-agent-data-plane /x-pack/metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. /x-pack/metricbeat/module/ @elastic/integrations diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index faeaa3d5968..e0d4d3d27ec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -33,6 +33,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Osquerybeat* +- Upgrade to osquery 5.10.2. {pull}37115[37115] *Packetbeat* @@ -114,6 +115,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix CassandraConnectionClosures metric configuration {pull}34742[34742] - Fix event mapping implementation for statsd module {pull}36925[36925] - The region and availability_zone ecs fields nested within the cloud field. {pull}37015[37015] +- Fix CPU and memory metrics collection from privileged process on Windows {issue}17314[17314]{pull}37027[37027] *Osquerybeat* diff --git a/NOTICE.txt b/NOTICE.txt index ea5514da6eb..ee78951e09d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13026,11 +13026,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.7.0 +Version: v0.8.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.7.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.8.1/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -21198,11 +21198,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/osquery/osquery-go -Version: v0.0.0-20230707154813-2e4891a0f444 +Version: v0.0.0-20231108163517-e3cde127e724 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/osquery/osquery-go@v0.0.0-20230707154813-2e4891a0f444/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/osquery/osquery-go@v0.0.0-20231108163517-e3cde127e724/LICENSE: MIT License diff --git a/go.mod b/go.mod index 55ef8b4acc5..56b9577786e 100644 --- a/go.mod +++ b/go.mod @@ -125,7 +125,7 @@ require ( github.com/mitchellh/hashstructure v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/osquery/osquery-go v0.0.0-20230707154813-2e4891a0f444 + github.com/osquery/osquery-go v0.0.0-20231108163517-e3cde127e724 github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 @@ -204,7 +204,7 @@ require ( github.com/elastic/elastic-agent-autodiscover v0.6.4 github.com/elastic/elastic-agent-libs v0.6.2 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 - github.com/elastic/elastic-agent-system-metrics v0.7.0 + github.com/elastic/elastic-agent-system-metrics v0.8.1 github.com/elastic/go-elasticsearch/v8 v8.10.0 github.com/elastic/mito v1.6.0 github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 diff --git a/go.sum b/go.sum index df5741e581b..a4b41b0f355 100644 --- a/go.sum +++ b/go.sum @@ -658,8 +658,8 @@ github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3 github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= -github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ= -github.com/elastic/elastic-agent-system-metrics v0.7.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= +github.com/elastic/elastic-agent-system-metrics v0.8.1 h1:eg6actuLeGJlIJFotHRdlAsz/3WhX2G8E0qI301IKBA= +github.com/elastic/elastic-agent-system-metrics v0.8.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= @@ -1566,8 +1566,8 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/osquery/osquery-go v0.0.0-20230707154813-2e4891a0f444 h1:UO3MEdZ4hkmAfhf7kXfuKR+e44gsHlEEsdWGOwZNLyQ= -github.com/osquery/osquery-go v0.0.0-20230707154813-2e4891a0f444/go.mod h1:mLJRc1Go8uP32LRALGvWj2lVJ+hDYyIfxDzVa+C5Yo8= +github.com/osquery/osquery-go v0.0.0-20231108163517-e3cde127e724 h1:z8XmnNQeCDZB3BwVoRxcqwo7MlDdsB6AJxqTap72S7w= +github.com/osquery/osquery-go v0.0.0-20231108163517-e3cde127e724/go.mod h1:mLJRc1Go8uP32LRALGvWj2lVJ+hDYyIfxDzVa+C5Yo8= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= diff --git a/libbeat/docs/howto/change-index-name.asciidoc b/libbeat/docs/howto/change-index-name.asciidoc index 8affe9887d0..c13d0621875 100644 --- a/libbeat/docs/howto/change-index-name.asciidoc +++ b/libbeat/docs/howto/change-index-name.asciidoc @@ -9,7 +9,7 @@ in the {es} output. You also need to configure the `setup.template.name` and ["source","sh",subs="attributes,callouts"] ----- output.elasticsearch.index: "customname-%{[{beat_version_key}]}" -setup.template.name: "customname" +setup.template.name: "customname-%{[{beat_version_key}]}" setup.template.pattern: "customname-%{[{beat_version_key}]}" ----- diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go index 9918654728e..1f428372b86 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go @@ -22,13 +22,14 @@ import ( "fmt" "net/http" + "github.com/elastic/elastic-agent-libs/logp" + awssdk "github.com/aws/aws-sdk-go-v2/aws" awscfg "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" conf "github.com/elastic/elastic-agent-libs/config" @@ -80,7 +81,6 @@ func fetchRawProviderMetadata( // LoadDefaultConfig loads the EC2 role credentials awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithHTTPClient(&client)) if err != nil { - logger.Warnf("error loading AWS default configuration: %s.", err) result.err = fmt.Errorf("failed loading AWS default configuration: %w", err) return } @@ -88,7 +88,6 @@ func fetchRawProviderMetadata( instanceIdentity, err := awsClient.GetInstanceIdentityDocument(context.TODO(), &imds.GetInstanceIdentityDocumentInput{}) if err != nil { - logger.Warnf("error fetching EC2 Identity Document: %s.", err) result.err = fmt.Errorf("failed fetching EC2 Identity Document: %w", err) return } @@ -96,13 +95,19 @@ func fetchRawProviderMetadata( // AWS Region must be set to be able to get EC2 Tags awsRegion := instanceIdentity.InstanceIdentityDocument.Region awsConfig.Region = awsRegion + accountID := instanceIdentity.InstanceIdentityDocument.AccountID clusterName, err := fetchEC2ClusterNameTag(awsConfig, instanceIdentity.InstanceIdentityDocument.InstanceID) if err != nil { logger.Warnf("error fetching cluster name metadata: %s.", err) - } + } else if clusterName != "" { + // for AWS cluster ID is used cluster ARN: arn:partition:service:region:account-id:resource-type/resource-id, example: + // arn:aws:eks:us-east-2:627286350134:cluster/cluster-name + clusterARN := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%v", awsRegion, accountID, clusterName) - accountID := instanceIdentity.InstanceIdentityDocument.AccountID + _, _ = result.metadata.Put("orchestrator.cluster.id", clusterARN) + _, _ = result.metadata.Put("orchestrator.cluster.name", clusterName) + } _, _ = result.metadata.Put("instance.id", instanceIdentity.InstanceIdentityDocument.InstanceID) _, _ = result.metadata.Put("machine.type", instanceIdentity.InstanceIdentityDocument.InstanceType) @@ -111,14 +116,6 @@ func fetchRawProviderMetadata( _, _ = result.metadata.Put("account.id", accountID) _, _ = result.metadata.Put("image.id", instanceIdentity.InstanceIdentityDocument.ImageID) - // for AWS cluster ID is used cluster ARN: arn:partition:service:region:account-id:resource-type/resource-id, example: - // arn:aws:eks:us-east-2:627286350134:cluster/cluster-name - if clusterName != "" { - clusterARN := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%v", awsRegion, accountID, clusterName) - - _, _ = result.metadata.Put("orchestrator.cluster.id", clusterARN) - _, _ = result.metadata.Put("orchestrator.cluster.name", clusterName) - } } func fetchEC2ClusterNameTag(awsConfig awssdk.Config, instanceID string) (string, error) { diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go index 2b9f0d90646..55e68f75607 100644 --- a/libbeat/processors/add_cloud_metadata/providers.go +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -101,7 +101,7 @@ func setupFetchers(providers map[string]provider, c *conf.C) ([]metadataFetcher, mf := make([]metadataFetcher, 0, len(providers)) visited := map[string]bool{} - // Iterate over all providers and create an unique meta-data fetcher per provider type. + // Iterate over all providers and create a unique meta-data fetcher per provider type. // Some providers might appear twice in the set of providers to support aliases on provider names. // For example aws and ec2 both use the same provider. // The loop tracks already seen providers in the `visited` set, to ensure that we do not create @@ -123,7 +123,7 @@ func setupFetchers(providers map[string]provider, c *conf.C) ([]metadataFetcher, } // fetchMetadata attempts to fetch metadata in parallel from each of the -// hosting providers supported by this processor. It wait for the results to +// hosting providers supported by this processor. It will wait for the results to // be returned or for a timeout to occur then returns the first result that // completed in time. func (p *addCloudMetadata) fetchMetadata() *result { @@ -169,6 +169,8 @@ func (p *addCloudMetadata) fetchMetadata() *result { // Bail out on first success. if result.err == nil && result.metadata != nil { return &result + } else if result.err != nil { + p.logger.Errorf("add_cloud_metadata: received error %v", result.err) } case <-ctx.Done(): p.logger.Debugf("add_cloud_metadata: timed-out waiting for all responses") diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 039e7299a8a..9303920b897 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -111,8 +111,9 @@ # cmdline is also part of the system process fields, but it may not be present # for some kernel level processes. fd is also part of the system process, but # is not available on all OSes and requires root to read for all processes. +# num_threads may not be readable for some privileged process on Windows, # cgroup is only available on linux. -SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state", "num_threads"] +SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state"] class Test(metricbeat.BaseTest): @@ -420,6 +421,9 @@ def test_process(self): found_cmdline = False for evt in output: process = evt["system"]["process"] + # Not all process will have 'cmdline' due to permission issues, + # especially on Windows. Therefore we ensure at least some of + # them will have it. found_cmdline |= "cmdline" in process # Remove 'env' prior to checking documented fields because its keys are dynamic. @@ -430,11 +434,13 @@ def test_process(self): process.pop("cgroup", None) process.pop("fd", None) process.pop("cmdline", None) + process.pop("num_threads", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) - - self.assertTrue( - found_cmdline, "cmdline not found in any process events") + # After iterating over all process, make sure at least one of them had + # the 'cmdline' set. + self.assertTrue( + found_cmdline, "cmdline not found in any process events") @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os") def test_process_unix(self): @@ -486,6 +492,7 @@ def test_process_unix(self): process.pop("cgroup", None) process.pop("cmdline", None) process.pop("fd", None) + process.pop("num_threads", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 1e421914cfb..f9ce5b7cb70 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-628b3b84-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-35e3b343-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-628b3b84-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-35e3b343-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-628b3b84-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-35e3b343-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/x-pack/osquerybeat/internal/distro/distro.go b/x-pack/osquerybeat/internal/distro/distro.go index d8d26536dbd..ac966173e1f 100644 --- a/x-pack/osquerybeat/internal/distro/distro.go +++ b/x-pack/osquerybeat/internal/distro/distro.go @@ -36,14 +36,14 @@ const ( osqueryCertsDarwinPath = "private/var/osquery/certs/" + osqueryCertsPEM osqueryCertsWindowsPath = "osquery/certs/" + osqueryCertsPEM - osqueryVersion = "5.8.2" + osqueryVersion = "5.10.2" osqueryMSIExt = ".msi" osqueryPkgExt = ".pkg" - osqueryDistroDarwinSHA256 = "1fea8ac9b603851d2e76c5fc73138a468a3075a3002c8cb1fd7fff53b889c4dd" - osqueryDistroLinuxSHA256 = "5bb2647b45a423e68d7dbc16ab2316c3f512d0944a56e4662c7010b59cddc721" - osqueryDistroLinuxARMSHA256 = "e51620928210970abb51d6ec79235bafff73bd354bdb54eec6e5969072d3d115" - osqueryDistroWindowsSHA256 = "d319837d4e95d1e477c2126d383501180925a29f488ff1164fa16d2e576f96dd" + osqueryDistroDarwinSHA256 = "a01d1f7da016f1e6bed54955e97982d491b7e55311433ff0fc985269160633af" + osqueryDistroLinuxSHA256 = "61ef2351a07dbc36ae9ebff605e8a7ecc4e09a07ac11f540d2aed78c143addbe" + osqueryDistroLinuxARMSHA256 = "106ea8a90dff0ccff852f44137848fe47ab9e8cfd27e5cd3a5ef963024b0564b" + osqueryDistroWindowsSHA256 = "f5a6955db724559638e43aef181e26eadfe4bfb827907ffd134d9abb0512cc58" ) type OSArch struct {