Osquerybeat: Upgrade osquery to version 5.0.1 (#28101) (#28150)

* Adjusted to changed packaging for MacOS and Linux official osquery
  distros. The MacOS osqueryd official distro now contains the application bundle osquery.app to support entitlement-based features.
* Corrected the osquerybeat reference configuration to reflect the
  latest changes. It was not up to date since 7.14.

(cherry picked from commit ab953d6)

Co-authored-by: Aleksandr Maus <aleksandr.maus@elastic.co>

x-pack/osquerybeat/_meta/config/beat.docker.yml.tmpl

@@ -6,11 +6,9 @@ osquerybeat:
 #  inputs:
 #    - type: osquery
 #      streams:
-#        - id: "E169F085-AC8B-48AF-9355-D2977030CE24"
-#          query: "select * from users"
 #        - id: "CFDE1EAA-0C6C-4D19-9EEC-45802B2A8C01"
 #          query: "select * from processes"
-#          interval: 1m
+#          interval: 60
 
 # ============================== Process Security ==============================
 # Disable seccomp system call filtering on Linux.

x-pack/osquerybeat/_meta/config/beat.reference.yml.tmpl

@@ -6,11 +6,9 @@ osquerybeat:
 #  inputs:
 #    - type: osquery
 #      streams:
-#        - id: "E169F085-AC8B-48AF-9355-D2977030CE24"
-#          query: "select * from users"
 #        - id: "CFDE1EAA-0C6C-4D19-9EEC-45802B2A8C01"
 #          query: "select * from processes"
-#          interval: 1m
+#          interval: 60
 
 # ============================== Process Security ==============================
 # Disable seccomp system call filtering on Linux.

x-pack/osquerybeat/_meta/config/beat.yml.tmpl

@@ -6,11 +6,9 @@ osquerybeat:
 #  inputs:
 #    - type: osquery
 #      streams:
-#        - id: "E169F085-AC8B-48AF-9355-D2977030CE24"
-#          query: "select * from users"
 #        - id: "CFDE1EAA-0C6C-4D19-9EEC-45802B2A8C01"
 #          query: "select * from processes"
-#          interval: 1m
+#          interval: 60
 
 # ============================== Process Security ==============================
 # Disable seccomp system call filtering on Linux.

x-pack/osquerybeat/beater/config_plugin.go

@@ -131,7 +131,7 @@ func (p *ConfigPlugin) GenerateConfig(ctx context.Context) (map[string]string, e
 		p.newQueryInfoMap = nil
 	}
 
-	p.log.Info("Osqueryd configuration:", c)
+	p.log.Debug("Osqueryd configuration:", c)
 
 	return map[string]string{
 		configName: c,

x-pack/osquerybeat/beater/install.go

@@ -64,6 +64,9 @@ func installOsqueryWithDir(ctx context.Context, dir string) error {
 	if installing {
 		// Check that osqueryd file is now installed
 		osqfn := distro.OsquerydFilename()
+		if runtime.GOOS == "darwin" {
+			osqfn = distro.OsquerydDarwinApp()
+		}
 		flog := log.With("file", osqfn)
 		exists, err := fileutil.FileExists(osqfn)
 		if err != nil {

x-pack/osquerybeat/internal/distro/distro.go

@@ -26,15 +26,18 @@ const (
 	osqueryDownloadBaseURL = "https://pkg.osquery.io"
 	osqueryName            = "osquery"
 	osqueryDName           = "osqueryd"
-	osqueryPath            = "usr/local/bin"
-	osqueryVersion         = "4.9.0"
-	osqueryMSIExt          = ".msi"
-	osqueryPkgExt          = ".pkg"
-
-	osqueryDistroDarwinSHA256   = "3f9ab772596f4da69687a2d7db9a382535b5eabf2346abd452b24666b8f25102"
-	osqueryDistroLinuxSHA256    = "4187f5b76e21ce96f765e509f96cb04708c43596b7609eba0e5a10ab5fdf58c5"
-	osqueryDistroLinuxARMSHA256 = "f3f5f3d6d81d727aad52a50b9f252aea3d7200add643eca06e9f462e66daeb18"
-	osqueryDistroWindowsSHA256  = "ae5e8b5948f3e2783aadc66e8b9d5d417b8606b39abf79f06af466c3455ce249"
+	osqueryDarwinApp       = "osquery.app"
+	osqueryDarwinPath      = "opt/osquery/lib/" + osqueryDarwinApp
+
+	osqueryLinuxPath = "opt/osquery/bin"
+	osqueryVersion   = "5.0.1"
+	osqueryMSIExt    = ".msi"
+	osqueryPkgExt    = ".pkg"
+
+	osqueryDistroDarwinSHA256   = "ec58996e64637d861ccead8dc6bc8865662728f6e5bc2694a3c92f0f4a371095"
+	osqueryDistroLinuxSHA256    = "acac95714d388f02d5f417b0aaf86de7dbb8f6b3788340a6f8517ee2cd314235"
+	osqueryDistroLinuxARMSHA256 = "712b704036929df14cbe5d3e41bd4e0ae325e698296691763af46dc0d6e77394"
+	osqueryDistroWindowsSHA256  = "e0d01f56e0739a0ce2b3beb03ecea277ed146754884e225cba45083043442acc"
 )
 
 type OSArch struct {
@@ -61,12 +64,20 @@ func OsquerydFilename() string {
 	return osqueryDName
 }
 
+func OsquerydDarwinApp() string {
+	return osqueryDarwinApp
+}
+
 func OsquerydPath(dir string) string {
 	return filepath.Join(dir, OsquerydFilename())
 }
 
-func OsquerydDistroPath() string {
-	return OsquerydPath(osqueryPath)
+func OsquerydDarwinDistroPath() string {
+	return osqueryDarwinPath
+}
+
+func OsquerydLinuxDistroPath() string {
+	return OsquerydPath(osqueryLinuxPath)
 }
 
 func OsquerydDistroFilename() string {

x-pack/osquerybeat/internal/install/install.go

@@ -23,7 +23,7 @@ import (
 )
 
 func InstallFromPkg(ctx context.Context, srcPkg, dstDir string, force bool) error {
-	dstfp := filepath.Join(dstDir, distro.OsquerydFilename())
+	dstfp := filepath.Join(dstDir, distro.OsquerydDarwinApp())
 
 	dir, err := installFromCommon(ctx, srcPkg, dstDir, dstfp, force, "pkgutil", "--expand-full", srcPkg)
 	// Remove the directory that was created could have been created by pkgutil
@@ -35,7 +35,7 @@ func InstallFromPkg(ctx context.Context, srcPkg, dstDir string, force bool) erro
 	}
 
 	// Copy over the osqueryd from under Payload into the dstDir directory
-	return devtools.Copy(path.Join(dir, "Payload", distro.OsquerydDistroPath()), dstfp)
+	return devtools.Copy(filepath.Join(dir, "Payload", distro.OsquerydDarwinDistroPath()), filepath.Join(dstDir, distro.OsquerydDarwinApp()))
 }
 
 func InstallFromMSI(ctx context.Context, srcMSI, dstDir string, force bool) error {
@@ -52,7 +52,7 @@ func InstallFromMSI(ctx context.Context, srcMSI, dstDir string, force bool) erro
 		return err
 	}
 
-	// Copy over the osqueryd from under osquery/osqueryd into the dstDir directory
+	// Copy over the or osquery.app osqueryd from under osquery/osqueryd into the dstDir directory
 	return devtools.Copy(path.Join(dir, "osquery", distro.OsquerydPath("osqueryd")), dstfp)
 }
 

x-pack/osquerybeat/internal/osqd/osqueryd.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -24,7 +25,8 @@ import (
 )
 
 const (
-	osqueryDName = "osqueryd"
+	osqueryDName               = "osqueryd"
+	osqueryDarwinAppBundlePath = "osquery.app/Contents/MacOS"
 )
 
 const (
@@ -351,6 +353,10 @@ func (q *OSQueryD) isVerbose() bool {
 }
 
 func osquerydPath(dir string) string {
+	if runtime.GOOS == "darwin" {
+		return filepath.Join(dir, osqueryDarwinAppBundlePath, osquerydFilename())
+
+	}
 	return filepath.Join(dir, osquerydFilename())
 }
 

x-pack/osquerybeat/osquerybeat.reference.yml

@@ -6,11 +6,9 @@ osquerybeat:
 #  inputs:
 #    - type: osquery
 #      streams:
-#        - id: "E169F085-AC8B-48AF-9355-D2977030CE24"
-#          query: "select * from users"
 #        - id: "CFDE1EAA-0C6C-4D19-9EEC-45802B2A8C01"
 #          query: "select * from processes"
-#          interval: 1m
+#          interval: 60
 
 # ============================== Process Security ==============================
 # Disable seccomp system call filtering on Linux.

x-pack/osquerybeat/osquerybeat.yml

@@ -6,11 +6,9 @@ osquerybeat:
 #  inputs:
 #    - type: osquery
 #      streams:
-#        - id: "E169F085-AC8B-48AF-9355-D2977030CE24"
-#          query: "select * from users"
 #        - id: "CFDE1EAA-0C6C-4D19-9EEC-45802B2A8C01"
 #          query: "select * from processes"
-#          interval: 1m
+#          interval: 60
 
 # ============================== Process Security ==============================
 # Disable seccomp system call filtering on Linux.

x-pack/osquerybeat/scripts/mage/distro.go

@@ -160,7 +160,7 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error {
 		}
 		defer os.RemoveAll(tmpdir)
 
-		osdp := distro.OsquerydDistroPath()
+		osdp := distro.OsquerydLinuxDistroPath()
 		if err := tar.ExtractFile(src, tmpdir, osdp); err != nil {
 			return err
 		}