Merge branch 'main' into ga/abs

.github/CODEOWNERS

@@ -50,7 +50,7 @@ CHANGELOG*
 /filebeat/module/santa @elastic/security-external-integrations
 /filebeat/module/system @elastic/elastic-agent-data-plane
 /filebeat/module/traefik @elastic/integrations
-/heartbeat/ @elastic/uptime
+/heartbeat/ @elastic/hosted-services
 /journalbeat @elastic/elastic-agent-data-plane
 /libbeat/ @elastic/elastic-agent-data-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
@@ -174,7 +174,7 @@ CHANGELOG*
 /x-pack/filebeat/module/zscaler @elastic/security-external-integrations
 /x-pack/filebeat/modules.d/zoom.yml.disabled @elastic/security-external-integrations
 /x-pack/filebeat/processors/decode_cef/ @elastic/security-external-integrations
-/x-pack/heartbeat/ @elastic/uptime
+/x-pack/heartbeat/ @elastic/hosted-services
 /x-pack/metricbeat/ @elastic/elastic-agent-data-plane
 /x-pack/metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
 /x-pack/metricbeat/module/ @elastic/integrations

CHANGELOG.next.asciidoc

@@ -33,6 +33,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Osquerybeat*
 
+- Upgrade to osquery 5.10.2. {pull}37115[37115]
 
 *Packetbeat*
 
@@ -114,6 +115,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix CassandraConnectionClosures metric configuration {pull}34742[34742]
 - Fix event mapping implementation for statsd module {pull}36925[36925]
 - The region and availability_zone ecs fields nested within the cloud field. {pull}37015[37015]
+- Fix CPU and memory metrics collection from privileged process on Windows {issue}17314[17314]{pull}37027[37027]
 
 *Osquerybeat*
 

NOTICE.txt

@@ -13026,11 +13026,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.7.0
+Version: v0.8.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.7.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.8.1/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -21198,11 +21198,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/osquery/osquery-go
-Version: v0.0.0-20230707154813-2e4891a0f444
+Version: v0.0.0-20231108163517-e3cde127e724
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/osquery/osquery-go@v0.0.0-20230707154813-2e4891a0f444/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/osquery/osquery-go@v0.0.0-20231108163517-e3cde127e724/LICENSE:
 
 MIT License
 

go.mod

@@ -125,7 +125,7 @@ require (
 	github.com/mitchellh/hashstructure v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/osquery/osquery-go v0.0.0-20230707154813-2e4891a0f444
+	github.com/osquery/osquery-go v0.0.0-20231108163517-e3cde127e724
 	github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
@@ -204,7 +204,7 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.6.4
 	github.com/elastic/elastic-agent-libs v0.6.2
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
-	github.com/elastic/elastic-agent-system-metrics v0.7.0
+	github.com/elastic/elastic-agent-system-metrics v0.8.1
 	github.com/elastic/go-elasticsearch/v8 v8.10.0
 	github.com/elastic/mito v1.6.0
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640

go.sum

@@ -658,8 +658,8 @@ github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3
 github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
-github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ=
-github.com/elastic/elastic-agent-system-metrics v0.7.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
+github.com/elastic/elastic-agent-system-metrics v0.8.1 h1:eg6actuLeGJlIJFotHRdlAsz/3WhX2G8E0qI301IKBA=
+github.com/elastic/elastic-agent-system-metrics v0.8.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
 github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
 github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
@@ -1566,8 +1566,8 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/osquery/osquery-go v0.0.0-20230707154813-2e4891a0f444 h1:UO3MEdZ4hkmAfhf7kXfuKR+e44gsHlEEsdWGOwZNLyQ=
-github.com/osquery/osquery-go v0.0.0-20230707154813-2e4891a0f444/go.mod h1:mLJRc1Go8uP32LRALGvWj2lVJ+hDYyIfxDzVa+C5Yo8=
+github.com/osquery/osquery-go v0.0.0-20231108163517-e3cde127e724 h1:z8XmnNQeCDZB3BwVoRxcqwo7MlDdsB6AJxqTap72S7w=
+github.com/osquery/osquery-go v0.0.0-20231108163517-e3cde127e724/go.mod h1:mLJRc1Go8uP32LRALGvWj2lVJ+hDYyIfxDzVa+C5Yo8=
 github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
 github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
 github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=

libbeat/docs/howto/change-index-name.asciidoc

@@ -9,7 +9,7 @@ in the {es} output. You also need to configure the `setup.template.name` and
 ["source","sh",subs="attributes,callouts"]
 -----
 output.elasticsearch.index: "customname-%{[{beat_version_key}]}"
-setup.template.name: "customname"
+setup.template.name: "customname-%{[{beat_version_key}]}"
 setup.template.pattern: "customname-%{[{beat_version_key}]}"
 -----
 

libbeat/processors/add_cloud_metadata/provider_aws_ec2.go

@@ -22,13 +22,14 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/elastic/elastic-agent-libs/logp"
+
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	awscfg "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
 
-	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 
 	conf "github.com/elastic/elastic-agent-libs/config"
@@ -80,29 +81,33 @@ func fetchRawProviderMetadata(
 	// LoadDefaultConfig loads the EC2 role credentials
 	awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithHTTPClient(&client))
 	if err != nil {
-		logger.Warnf("error loading AWS default configuration: %s.", err)
 		result.err = fmt.Errorf("failed loading AWS default configuration: %w", err)
 		return
 	}
 	awsClient := NewIMDSClient(awsConfig)
 
 	instanceIdentity, err := awsClient.GetInstanceIdentityDocument(context.TODO(), &imds.GetInstanceIdentityDocumentInput{})
 	if err != nil {
-		logger.Warnf("error fetching EC2 Identity Document: %s.", err)
 		result.err = fmt.Errorf("failed fetching EC2 Identity Document: %w", err)
 		return
 	}
 
 	// AWS Region must be set to be able to get EC2 Tags
 	awsRegion := instanceIdentity.InstanceIdentityDocument.Region
 	awsConfig.Region = awsRegion
+	accountID := instanceIdentity.InstanceIdentityDocument.AccountID
 
 	clusterName, err := fetchEC2ClusterNameTag(awsConfig, instanceIdentity.InstanceIdentityDocument.InstanceID)
 	if err != nil {
 		logger.Warnf("error fetching cluster name metadata: %s.", err)
-	}
+	} else if clusterName != "" {
+		// for AWS cluster ID is used cluster ARN: arn:partition:service:region:account-id:resource-type/resource-id, example:
+		// arn:aws:eks:us-east-2:627286350134:cluster/cluster-name
+		clusterARN := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%v", awsRegion, accountID, clusterName)
 
-	accountID := instanceIdentity.InstanceIdentityDocument.AccountID
+		_, _ = result.metadata.Put("orchestrator.cluster.id", clusterARN)
+		_, _ = result.metadata.Put("orchestrator.cluster.name", clusterName)
+	}
 
 	_, _ = result.metadata.Put("instance.id", instanceIdentity.InstanceIdentityDocument.InstanceID)
 	_, _ = result.metadata.Put("machine.type", instanceIdentity.InstanceIdentityDocument.InstanceType)
@@ -111,14 +116,6 @@ func fetchRawProviderMetadata(
 	_, _ = result.metadata.Put("account.id", accountID)
 	_, _ = result.metadata.Put("image.id", instanceIdentity.InstanceIdentityDocument.ImageID)
 
-	// for AWS cluster ID is used cluster ARN: arn:partition:service:region:account-id:resource-type/resource-id, example:
-	// arn:aws:eks:us-east-2:627286350134:cluster/cluster-name
-	if clusterName != "" {
-		clusterARN := fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%v", awsRegion, accountID, clusterName)
-
-		_, _ = result.metadata.Put("orchestrator.cluster.id", clusterARN)
-		_, _ = result.metadata.Put("orchestrator.cluster.name", clusterName)
-	}
 }
 
 func fetchEC2ClusterNameTag(awsConfig awssdk.Config, instanceID string) (string, error) {

libbeat/processors/add_cloud_metadata/providers.go

@@ -101,7 +101,7 @@ func setupFetchers(providers map[string]provider, c *conf.C) ([]metadataFetcher,
 	mf := make([]metadataFetcher, 0, len(providers))
 	visited := map[string]bool{}
 
-	// Iterate over all providers and create an unique meta-data fetcher per provider type.
+	// Iterate over all providers and create a unique meta-data fetcher per provider type.
 	// Some providers might appear twice in the set of providers to support aliases on provider names.
 	// For example aws and ec2 both use the same provider.
 	// The loop tracks already seen providers in the `visited` set, to ensure that we do not create
@@ -123,7 +123,7 @@ func setupFetchers(providers map[string]provider, c *conf.C) ([]metadataFetcher,
 }
 
 // fetchMetadata attempts to fetch metadata in parallel from each of the
-// hosting providers supported by this processor. It wait for the results to
+// hosting providers supported by this processor. It will wait for the results to
 // be returned or for a timeout to occur then returns the first result that
 // completed in time.
 func (p *addCloudMetadata) fetchMetadata() *result {
@@ -169,6 +169,8 @@ func (p *addCloudMetadata) fetchMetadata() *result {
 			// Bail out on first success.
 			if result.err == nil && result.metadata != nil {
 				return &result
+			} else if result.err != nil {
+				p.logger.Errorf("add_cloud_metadata: received error %v", result.err)
 			}
 		case <-ctx.Done():
 			p.logger.Debugf("add_cloud_metadata: timed-out waiting for all responses")

metricbeat/module/system/test_system.py

@@ -111,8 +111,9 @@
 # cmdline is also part of the system process fields, but it may not be present
 # for some kernel level processes. fd is also part of the system process, but
 # is not available on all OSes and requires root to read for all processes.
+# num_threads may not be readable for some privileged process on Windows,
 # cgroup is only available on linux.
-SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state", "num_threads"]
+SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state"]
 
 
 class Test(metricbeat.BaseTest):
@@ -420,6 +421,9 @@ def test_process(self):
         found_cmdline = False
         for evt in output:
             process = evt["system"]["process"]
+            # Not all process will have 'cmdline' due to permission issues,
+            # especially on Windows. Therefore we ensure at least some of
+            # them will have it.
             found_cmdline |= "cmdline" in process
 
             # Remove 'env' prior to checking documented fields because its keys are dynamic.
@@ -430,11 +434,13 @@ def test_process(self):
             process.pop("cgroup", None)
             process.pop("fd", None)
             process.pop("cmdline", None)
+            process.pop("num_threads", None)
 
             self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys())
-
-            self.assertTrue(
-                found_cmdline, "cmdline not found in any process events")
+        # After iterating over all process, make sure at least one of them had
+        # the 'cmdline' set.
+        self.assertTrue(
+            found_cmdline, "cmdline not found in any process events")
 
     @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os")
     def test_process_unix(self):
@@ -486,6 +492,7 @@ def test_process_unix(self):
             process.pop("cgroup", None)
             process.pop("cmdline", None)
             process.pop("fd", None)
+            process.pop("num_threads", None)
 
             self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys())
 

testing/environments/snapshot.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-628b3b84-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-35e3b343-SNAPSHOT
     # When extend is used it merges healthcheck.tests, see:
     # https://github.com/docker/compose/issues/8962
     # healthcheck:
@@ -31,7 +31,7 @@ services:
     - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:8.12.0-628b3b84-SNAPSHOT
+    image: docker.elastic.co/logstash/logstash:8.12.0-35e3b343-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 600
@@ -44,7 +44,7 @@ services:
       - 5055:5055
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.12.0-628b3b84-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.12.0-35e3b343-SNAPSHOT
     environment:
     - "ELASTICSEARCH_USERNAME=kibana_system_user"
     - "ELASTICSEARCH_PASSWORD=testing"

x-pack/osquerybeat/internal/distro/distro.go

@@ -36,14 +36,14 @@ const (
 	osqueryCertsDarwinPath  = "private/var/osquery/certs/" + osqueryCertsPEM
 	osqueryCertsWindowsPath = "osquery/certs/" + osqueryCertsPEM
 
-	osqueryVersion = "5.8.2"
+	osqueryVersion = "5.10.2"
 	osqueryMSIExt  = ".msi"
 	osqueryPkgExt  = ".pkg"
 
-	osqueryDistroDarwinSHA256   = "1fea8ac9b603851d2e76c5fc73138a468a3075a3002c8cb1fd7fff53b889c4dd"
-	osqueryDistroLinuxSHA256    = "5bb2647b45a423e68d7dbc16ab2316c3f512d0944a56e4662c7010b59cddc721"
-	osqueryDistroLinuxARMSHA256 = "e51620928210970abb51d6ec79235bafff73bd354bdb54eec6e5969072d3d115"
-	osqueryDistroWindowsSHA256  = "d319837d4e95d1e477c2126d383501180925a29f488ff1164fa16d2e576f96dd"
+	osqueryDistroDarwinSHA256   = "a01d1f7da016f1e6bed54955e97982d491b7e55311433ff0fc985269160633af"
+	osqueryDistroLinuxSHA256    = "61ef2351a07dbc36ae9ebff605e8a7ecc4e09a07ac11f540d2aed78c143addbe"
+	osqueryDistroLinuxARMSHA256 = "106ea8a90dff0ccff852f44137848fe47ab9e8cfd27e5cd3a5ef963024b0564b"
+	osqueryDistroWindowsSHA256  = "f5a6955db724559638e43aef181e26eadfe4bfb827907ffd134d9abb0512cc58"
 )
 
 type OSArch struct {