[Auditbeat] Cherry-pick elastic#9732 to 6.6: User metricset: Fetch gr…

…oups by user (elastic#9872)

Cherry-pick of PR elastic#9732 to 6.6 branch. Original message: 

Changes the user metricset to looking up groups by user instead of users by groups.

Also changes the types of the system.audit.user.uid and system.audit.user.gid fields from integer to keyword to accommodate Windows in the future.

Fixes elastic#9679.

auditbeat/docs/fields.asciidoc

@@ -3821,7 +3821,7 @@ User name.
 *`system.audit.user.uid`*::
 +
 --
-type: integer
+type: keyword
 
 User ID.
 
@@ -3831,7 +3831,7 @@ User ID.
 *`system.audit.user.gid`*::
 +
 --
-type: integer
+type: keyword
 
 Group ID.
 

x-pack/auditbeat/module/system/user/_meta/data.json

@@ -7,7 +7,7 @@
     "event": {
         "action": "existing_user",
         "dataset": "user",
-        "id": "b0bbc4e2-9540-4aaf-b74e-ef9de506e852",
+        "id": "11b3b49c-79a1-4983-aea9-3257a3073a71",
         "kind": "state",
         "module": "system"
     },
@@ -19,14 +19,14 @@
         "audit": {
             "user": {
                 "dir": "/home/elastic",
-                "gid": 1002,
+                "gid": "1002",
                 "group": [
                     {
-                        "gid": 1002,
+                        "gid": "1002",
                         "name": "elastic"
                     },
                     {
-                        "gid": 999,
+                        "gid": "999",
                         "name": "docker"
                     }
                 ],
@@ -36,13 +36,13 @@
                     "type": "shadow_password"
                 },
                 "shell": "/usr/bin/zsh",
-                "uid": 1002,
+                "uid": "1002",
                 "user_information": ",,,"
             }
         }
     },
     "user": {
-        "id": 1002,
+        "id": "1002",
         "name": "elastic"
     }
-}
+}

x-pack/auditbeat/module/system/user/_meta/fields.yml

@@ -9,11 +9,11 @@
       description: >
           User name.
     - name: uid
-      type: integer
+      type: keyword
       description: >
           User ID.
     - name: gid
-      type: integer
+      type: keyword
       description: >
           Group ID.
     - name: dir

x-pack/auditbeat/module/system/user/user.go

@@ -12,8 +12,8 @@ import (
 	"encoding/gob"
 	"fmt"
 	"io"
+	"os/user"
 	"runtime"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -105,21 +105,15 @@ type User struct {
 	PasswordType     passwordType
 	PasswordChanged  time.Time
 	PasswordHashHash []byte
-	UID              uint32
-	GID              uint32
-	Groups           []Group
+	UID              string
+	GID              string
+	Groups           []*user.Group
 	UserInfo         string
 	Dir              string
 	Shell            string
 	Action           string
 }
 
-// Group contains information about a group.
-type Group struct {
-	Name string
-	GID  uint32
-}
-
 // Hash creates a hash for User.
 func (user User) Hash() uint64 {
 	h := xxhash.New64()
@@ -128,14 +122,14 @@ func (user User) Hash() uint64 {
 	binary.Write(h, binary.BigEndian, uint8(user.PasswordType))
 	h.WriteString(user.PasswordChanged.String())
 	h.Write(user.PasswordHashHash)
-	h.WriteString(strconv.Itoa(int(user.UID)))
-	h.WriteString(strconv.Itoa(int(user.GID)))
+	h.WriteString(user.UID)
+	h.WriteString(user.GID)
 	h.WriteString(user.Dir)
 	h.WriteString(user.Shell)
 
 	for _, group := range user.Groups {
 		h.WriteString(group.Name)
-		h.WriteString(strconv.Itoa(int(group.GID)))
+		h.WriteString(group.Gid)
 	}
 
 	return h.Sum64()
@@ -167,7 +161,7 @@ func (user User) toMapStr() common.MapStr {
 		for _, group := range user.Groups {
 			groupMapStr = append(groupMapStr, common.MapStr{
 				"name": group.Name,
-				"gid":  group.GID,
+				"gid":  group.Gid,
 			})
 		}
 		evt.Put("group", groupMapStr)
@@ -347,7 +341,7 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error {
 
 	if len(newInCache) > 0 && len(missingFromCache) > 0 {
 		// Check for changes to users
-		missingUserMap := make(map[uint32](*User))
+		missingUserMap := make(map[string](*User))
 		for _, missingUser := range missingFromCache {
 			missingUserMap[missingUser.(*User).UID] = missingUser.(*User)
 		}
@@ -441,7 +435,7 @@ func userMessage(user *User, action eventAction) string {
 		actionString, user.Name, user.UID, fmtGroups(user.Groups))
 }
 
-func fmtGroups(groups []Group) string {
+func fmtGroups(groups []*user.Group) string {
 	var b strings.Builder
 
 	b.WriteString(groups[0].Name)

x-pack/auditbeat/module/system/user/user_test.go

@@ -14,8 +14,6 @@ import (
 )
 
 func TestData(t *testing.T) {
-	t.Skip("test is failing in CI")
-
 	f := mbtest.NewReportingMetricSetV2(t, getConfig())
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {

x-pack/auditbeat/module/system/user/users_linux.go

@@ -8,15 +8,15 @@ package user
 
 // #include <sys/types.h>
 // #include <pwd.h>
-// #include <grp.h>
 // #include <shadow.h>
 import "C"
 
 import (
 	"crypto/sha512"
+	"os/user"
+	"strconv"
 	"strings"
 	"time"
-	"unsafe"
 
 	"github.com/pkg/errors"
 )
@@ -62,8 +62,8 @@ func readPasswdFile(readPasswords bool) ([]*User, error) {
 		// passwd is C.struct_passwd
 		user := &User{
 			Name:     C.GoString(passwd.pw_name),
-			UID:      uint32(passwd.pw_uid),
-			GID:      uint32(passwd.pw_gid),
+			UID:      strconv.Itoa(int(passwd.pw_uid)),
+			GID:      strconv.Itoa(int(passwd.pw_gid)),
 			UserInfo: C.GoString(passwd.pw_gecos),
 			Dir:      C.GoString(passwd.pw_dir),
 			Shell:    C.GoString(passwd.pw_shell),
@@ -92,20 +92,31 @@ func readPasswdFile(readPasswords bool) ([]*User, error) {
 }
 
 func enrichWithGroups(users []*User) error {
-	gidToGroup, userToGroup, err := readGroupFile()
-	if err != nil {
-		return err
-	}
+	gidCache := make(map[string]*user.Group, len(users))
 
-	for _, user := range users {
-		primaryGroup, found := gidToGroup[user.GID]
-		if found {
-			user.Groups = append(user.Groups, primaryGroup)
+	for _, u := range users {
+		goUser := user.User{
+			Uid:      u.UID,
+			Gid:      u.GID,
+			Username: u.Name,
 		}
 
-		secondaryGroups, found := userToGroup[user.Name]
-		if found {
-			user.Groups = append(user.Groups, secondaryGroups...)
+		groupIds, err := goUser.GroupIds()
+		if err != nil {
+			return errors.Wrapf(err, "error getting group IDs for user %v (UID: %v)", u.Name, u.UID)
+		}
+
+		for _, gid := range groupIds {
+			group, found := gidCache[gid]
+			if !found {
+				group, err = user.LookupGroupId(gid)
+				if err != nil {
+					return errors.Wrapf(err, "error looking up group ID %v for user %v (UID: %v)", gid, u.Name, u.UID)
+				}
+				gidCache[gid] = group
+			}
+
+			u.Groups = append(u.Groups, group)
 		}
 	}
 
@@ -138,51 +149,6 @@ func enrichWithShadow(users []*User) error {
 	return nil
 }
 
-// readGroupFile reads /etc/group and returns two maps:
-// The first maps group IDs to groups.
-// The second maps group members (user names) to groups.
-// See getgrent(3) for details of the structs.
-func readGroupFile() (map[uint32]Group, map[string][]Group, error) {
-	C.setgrent()
-	defer C.endgrent()
-
-	groupIDMap := make(map[uint32]Group)
-	groupMemberMap := make(map[string][]Group)
-	for cgroup, err := C.getgrent(); cgroup != nil; cgroup, err = C.getgrent() {
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "error while reading group file")
-		}
-
-		groupName := C.GoString(cgroup.gr_name)
-		gid := uint32(cgroup.gr_gid)
-
-		group := Group{
-			Name: groupName,
-			GID:  gid,
-		}
-
-		groupIDMap[gid] = group
-
-		/*
-			group.gr_mem is a NULL-terminated array of pointers to user names (char **)
-			which makes some pointer arithmetic necessary to read it.
-		*/
-		for i := 0; ; i++ {
-			offset := (unsafe.Sizeof(unsafe.Pointer(*cgroup.gr_mem)) * uintptr(i))
-			member := *(**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(cgroup.gr_mem)) + offset))
-
-			if member == nil {
-				break
-			}
-
-			groupMember := C.GoString(member)
-			groupMemberMap[groupMember] = append(groupMemberMap[groupMember], group)
-		}
-	}
-
-	return groupIDMap, groupMemberMap, nil
-}
-
 // shadowFileEntry represents an entry in /etc/shadow. See getspnam(3) for details.
 type shadowFileEntry struct {
 	LastChanged time.Time

x-pack/auditbeat/tests/system/test_metricsets.py

@@ -45,7 +45,6 @@ def test_metricset_socket(self):
         self.check_metricset("system", "socket", COMMON_FIELDS + fields, warnings_allowed=True)
 
     @unittest.skipUnless(sys.platform == "linux2", "Only implemented for Linux")
-    @unittest.skip("Test is failing in CI")  # https://github.com/elastic/beats/issues/9679
     def test_metricset_user(self):
         """
         user metricset collects information about users on a server.