x-pack/filebeat/input/entityanalytics/provider/okta: add support for …

…role and factor data collection (#41460)

(cherry picked from commit 34d7c4f)

CHANGELOG.next.asciidoc

@@ -328,6 +328,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206]
 - Improved Azure Blob Storage input documentation. {pull}41252[41252]
 - Make ETW input GA. {pull}41389[41389]
+- Add support for Okta entity analytics provider to collect role and factor data for users. {pull}41460[41460]
 
 *Auditbeat*
 

x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc

@@ -959,6 +959,7 @@ Example configuration:
   id: okta-1
   provider: okta
   dataset: "all"
+  enrich_with: ["groups", "roles"]
   sync_interval: "12h"
   update_interval: "30m"
   okta_domain: "OKTA_DOMAIN"
@@ -992,6 +993,13 @@ or may be left empty for the default behavior which is to collect all entities.
 When the `dataset` is set to "devices", some user entity data is collected in order
 to populate the registered users and registered owner fields for each device.
 
+[float]
+===== `enrich_with`
+
+The metadata to enrich users with. This is an array of values that may contain
+"groups", "roles" and "factors", or "none". If the array only contains "none", no
+metadata is collected for users. The default behavior is to collect "groups".
+
 [float]
 ===== `sync_interval`
 

x-pack/filebeat/input/entityanalytics/provider/okta/conf.go

@@ -23,6 +23,7 @@ func defaultConfig() conf {
 	transport.Timeout = 30 * time.Second
 
 	return conf{
+		EnrichWith:     []string{"groups"},
 		SyncInterval:   24 * time.Hour,
 		UpdateInterval: 15 * time.Minute,
 		LimitWindow:    time.Minute,
@@ -48,6 +49,12 @@ type conf struct {
 	// the API. It can be ""/"all", "users", or
 	// "devices".
 	Dataset string `config:"dataset"`
+	// EnrichWith specifies the additional data that
+	// will be used to enrich user data. It can include
+	// "groups", "roles" and "factors".
+	// If it is a single element with "none", no
+	// enrichment is performed.
+	EnrichWith []string `config:"enrich_with"`
 
 	// SyncInterval is the time between full
 	// synchronisation operations.

x-pack/filebeat/input/entityanalytics/provider/okta/okta.go

@@ -15,6 +15,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
@@ -460,15 +461,15 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn
 
 		if fullSync {
 			for _, u := range batch {
-				p.addGroup(ctx, u, state)
+				p.addUserMetadata(ctx, u, state)
 				if u.LastUpdated.After(lastUpdated) {
 					lastUpdated = u.LastUpdated
 				}
 			}
 		} else {
 			users = grow(users, len(batch))
 			for _, u := range batch {
-				su := p.addGroup(ctx, u, state)
+				su := p.addUserMetadata(ctx, u, state)
 				users = append(users, su)
 				if u.LastUpdated.After(lastUpdated) {
 					lastUpdated = u.LastUpdated
@@ -500,14 +501,41 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn
 	return users, nil
 }
 
-func (p *oktaInput) addGroup(ctx context.Context, u okta.User, state *stateStore) *User {
+func (p *oktaInput) addUserMetadata(ctx context.Context, u okta.User, state *stateStore) *User {
 	su := state.storeUser(u)
-	groups, _, err := okta.GetUserGroupDetails(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger)
-	if err != nil {
-		p.logger.Warnf("failed to get user group membership for %s: %v", u.ID, err)
+	switch len(p.cfg.EnrichWith) {
+	case 1:
+		if p.cfg.EnrichWith[0] != "none" {
+			break
+		}
+		fallthrough
+	case 0:
 		return su
 	}
-	su.Groups = groups
+	if slices.Contains(p.cfg.EnrichWith, "groups") {
+		groups, _, err := okta.GetUserGroupDetails(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger)
+		if err != nil {
+			p.logger.Warnf("failed to get user group membership for %s: %v", u.ID, err)
+		} else {
+			su.Groups = groups
+		}
+	}
+	if slices.Contains(p.cfg.EnrichWith, "factors") {
+		factors, _, err := okta.GetUserFactors(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger)
+		if err != nil {
+			p.logger.Warnf("failed to get user factors for %s: %v", u.ID, err)
+		} else {
+			su.Factors = factors
+		}
+	}
+	if slices.Contains(p.cfg.EnrichWith, "roles") {
+		roles, _, err := okta.GetUserRoles(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger)
+		if err != nil {
+			p.logger.Warnf("failed to get user roles for %s: %v", u.ID, err)
+		} else {
+			su.Roles = roles
+		}
+	}
 	return su
 }
 

x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go

@@ -13,6 +13,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"path"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -31,13 +32,14 @@ func TestOktaDoFetch(t *testing.T) {
 
 	tests := []struct {
 		dataset     string
+		enrichWith  []string
 		wantUsers   bool
 		wantDevices bool
 	}{
-		{dataset: "", wantUsers: true, wantDevices: true},
-		{dataset: "all", wantUsers: true, wantDevices: true},
-		{dataset: "users", wantUsers: true, wantDevices: false},
-		{dataset: "devices", wantUsers: false, wantDevices: true},
+		{dataset: "", enrichWith: []string{"groups"}, wantUsers: true, wantDevices: true},
+		{dataset: "all", enrichWith: []string{"groups"}, wantUsers: true, wantDevices: true},
+		{dataset: "users", enrichWith: []string{"groups", "roles", "factors"}, wantUsers: true, wantDevices: false},
+		{dataset: "devices", enrichWith: []string{"groups"}, wantUsers: false, wantDevices: true},
 	}
 
 	for _, test := range tests {
@@ -56,14 +58,18 @@ func TestOktaDoFetch(t *testing.T) {
 				window  = time.Minute
 				key     = "token"
 				users   = `[{"id":"USERID","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/USERID"}}}]`
+				roles   = `[{"id":"IFIFAX2BIRGUSTQ","label":"Application administrator","type":"APP_ADMIN","status":"ACTIVE","created":"2019-02-06T16:17:40.000Z","lastUpdated":"2019-02-06T16:17:40.000Z","assignmentType":"USER"},{"id":"JBCUYUC7IRCVGS27IFCE2SKO","label":"Help Desk administrator","type":"HELP_DESK_ADMIN","status":"ACTIVE","created":"2019-02-06T16:17:40.000Z","lastUpdated":"2019-02-06T16:17:40.000Z","assignmentType":"USER"},{"id":"ra125eqBFpETrMwu80g4","label":"Organization administrator","type":"ORG_ADMIN","status":"ACTIVE","created":"2019-02-06T16:17:40.000Z","lastUpdated":"2019-02-06T16:17:40.000Z","assignmentType":"USER"},{"id":"gra25fapn1prGTBKV0g4","label":"API Access Management administrator","type":"API_ACCESS_MANAGEMENT_ADMIN","status":"ACTIVE","created\"":"2019-02-06T16:20:57.000Z","lastUpdated\"":"2019-02-06T16:20:57.000Z","assignmentType\"":"GROUP"}]`
 				groups  = `[{"id":"USERID","profile":{"description":"All users in your organization","name":"Everyone"}}]`
+				factors = `[{"id":"ufs2bysphxKODSZKWVCT","factorType":"question","provider":"OKTA","vendorName":"OKTA","status":"ACTIVE","created":"2014-04-15T18:10:06.000Z","lastUpdated":"2014-04-15T18:10:06.000Z","profile":{"question":"favorite_art_piece","questionText":"What is your favorite piece of art?"}},{"id":"ostf2gsyictRQDSGTDZE","factorType":"token:software:totp","provider":"OKTA","status":"PENDING_ACTIVATION","created":"2014-06-27T20:27:33.000Z","lastUpdated":"2014-06-27T20:27:33.000Z","profile":{"credentialId":"dade.murphy@example.com"}},{"id":"sms2gt8gzgEBPUWBIFHN","factorType":"sms","provider":"OKTA","status":"ACTIVE","created":"2014-06-27T20:27:26.000Z","lastUpdated":"2014-06-27T20:27:26.000Z","profile":{"phoneNumber":"+1-555-415-1337"}}]`
 				devices = `[{"id":"DEVICEID","status":"STATUS","created":"2019-10-02T18:03:07.000Z","lastUpdated":"2019-10-02T18:03:07.000Z","profile":{"displayName":"Example Device name 1","platform":"WINDOWS","serialNumber":"XXDDRFCFRGF3M8MD6D","sid":"S-1-11-111","registered":true,"secureHardwarePresent":false,"diskEncryptionType":"ALL_INTERNAL_VOLUMES"},"resourceType":"UDDevice","resourceDisplayName":{"value":"Example Device name 1","sensitive":false},"resourceAlternateId":null,"resourceId":"DEVICEID","_links":{"activate":{"href":"https://localhost/api/v1/devices/DEVICEID/lifecycle/activate","hints":{"allow":["POST"]}},"self":{"href":"https://localhost/api/v1/devices/DEVICEID","hints":{"allow":["GET","PATCH","PUT"]}},"users":{"href":"https://localhost/api/v1/devices/DEVICEID/users","hints":{"allow":["GET"]}}}}]`
 			)
 
 			data := map[string]string{
 				"users":   users,
+				"roles":   roles,
 				"groups":  groups,
 				"devices": devices,
+				"factors": factors,
 			}
 
 			var wantUsers []User
@@ -88,29 +94,50 @@ func TestOktaDoFetch(t *testing.T) {
 					t.Fatalf("failed to unmarshal device data: %v", err)
 				}
 			}
+			var wantFactors []okta.Factor
+			if slices.Contains(test.enrichWith, "factors") {
+				err := json.Unmarshal([]byte(factors), &wantFactors)
+				if err != nil {
+					t.Fatalf("failed to unmarshal factor data: %v", err)
+				}
+			}
+			var wantRoles []okta.Role
+			if slices.Contains(test.enrichWith, "roles") {
+				err := json.Unmarshal([]byte(roles), &wantRoles)
+				if err != nil {
+					t.Fatalf("failed to unmarshal role data: %v", err)
+				}
+			}
 
 			wantStates := make(map[string]State)
 
 			// Set the number of repeats.
 			const repeats = 3
 			var n int
-			ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			setHeaders := func(w http.ResponseWriter) {
 				// Leave 49 remaining, reset in one minute.
 				w.Header().Add("x-rate-limit-limit", "50")
 				w.Header().Add("x-rate-limit-remaining", "49")
 				w.Header().Add("x-rate-limit-reset", fmt.Sprint(time.Now().Add(time.Minute).Unix()))
-
-				if strings.HasPrefix(r.URL.Path, "/api/v1/users") && strings.HasSuffix(r.URL.Path, "groups") {
-					// Give the groups if this is a get user groups request.
-					userid := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/users/"), "/groups")
-					fmt.Fprintln(w, strings.ReplaceAll(data["groups"], "USERID", userid))
-					return
-				}
-				if strings.HasPrefix(r.URL.Path, "/api/v1/device") && strings.HasSuffix(r.URL.Path, "users") {
-					// Give one user if this is a get device users request.
-					fmt.Fprintln(w, data["users"])
+			}
+			mux := http.NewServeMux()
+			mux.Handle("/api/v1/users/{userid}/{metadata}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				setHeaders(w)
+				attr := r.PathValue("metadata")
+				if attr != "groups" {
+					fmt.Fprintln(w, data[attr])
 					return
 				}
+				// Give the groups if this is a get user groups request.
+				userid := r.PathValue("userid")
+				fmt.Fprintln(w, strings.ReplaceAll(data[attr], "USERID", userid))
+			}))
+			mux.Handle("/api/v1/devices/{deviceid}/users", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				setHeaders(w)
+				fmt.Fprintln(w, data["users"])
+			}))
+			mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				setHeaders(w)
 
 				base := path.Base(r.URL.Path)
 
@@ -143,6 +170,7 @@ func TestOktaDoFetch(t *testing.T) {
 				)
 				fmt.Fprintln(w, replacer.Replace(data[base]))
 			}))
+			ts := httptest.NewTLSServer(mux)
 			defer ts.Close()
 
 			u, err := url.Parse(ts.URL)
@@ -154,6 +182,7 @@ func TestOktaDoFetch(t *testing.T) {
 					OktaDomain: u.Host,
 					OktaToken:  key,
 					Dataset:    test.dataset,
+					EnrichWith: test.enrichWith,
 				},
 				client: ts.Client(),
 				lim:    rate.NewLimiter(1, 1),
@@ -196,6 +225,12 @@ func TestOktaDoFetch(t *testing.T) {
 					if g.ID != wantID {
 						t.Errorf("unexpected user ID for user %d: got:%s want:%s", i, g.ID, wantID)
 					}
+					if len(g.Factors) != len(wantFactors) {
+						t.Errorf("number of factors for user %d: got:%d want:%d", i, len(g.Factors), len(wantFactors))
+					}
+					if len(g.Roles) != len(wantRoles) {
+						t.Errorf("number of roles for user %d: got:%d want:%d", i, len(g.Roles), len(wantRoles))
+					}
 					for j, gg := range g.Groups {
 						if gg.ID != wantID {
 							t.Errorf("unexpected used ID for user group %d in %d: got:%s want:%s", j, i, gg.ID, wantID)

x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go

@@ -37,8 +37,10 @@ const (
 
 type User struct {
 	okta.User `json:"properties"`
-	Groups    []okta.Group `json:"groups"`
-	State     State        `json:"state"`
+	Groups    []okta.Group  `json:"groups"`
+	Roles     []okta.Role   `json:"roles"`
+	Factors   []okta.Factor `json:"factors"`
+	State     State         `json:"state"`
 }
 
 type Device struct {