From 7d7d8a3ed43fb568317091eca9c5191485d6de91 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 28 Oct 2024 13:24:15 +1030 Subject: [PATCH 1/2] x-pack/filebeat/input/entityanalytics/provider/okta: add support for role and factor data collection --- CHANGELOG.next.asciidoc | 1 + .../inputs/input-entity-analytics.asciidoc | 8 +++ .../entityanalytics/provider/okta/conf.go | 7 ++ .../entityanalytics/provider/okta/okta.go | 42 ++++++++++-- .../provider/okta/okta_test.go | 65 ++++++++++++++----- .../provider/okta/statestore.go | 6 +- 6 files changed, 105 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c1084282c1b3..214454fb6d41 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -328,6 +328,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206] - Improved Azure Blob Storage input documentation. {pull}41252[41252] - Make ETW input GA. {pull}41389[41389] +- Add support for Okta entity analytics provider to collect role and factor data for users. {pull}41460[41460] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc index 5ca419acd3e6..b4b701d3919d 100644 --- a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc @@ -959,6 +959,7 @@ Example configuration: id: okta-1 provider: okta dataset: "all" + enrich_with: ["groups", "roles"] sync_interval: "12h" update_interval: "30m" okta_domain: "OKTA_DOMAIN" @@ -992,6 +993,13 @@ or may be left empty for the default behavior which is to collect all entities. When the `dataset` is set to "devices", some user entity data is collected in order to populate the registered users and registered owner fields for each device. +[float] +===== `enrich_with` + +The metadata to enrich users with. This is an array of values that may contain +"groups", "roles" and "factors", or "none". If the array only contains "none", no +metadata is collected for users. The default behavior is to collect "groups". + [float] ===== `sync_interval` diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go index 2bab4c9e67d2..41a3895a70d2 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go @@ -23,6 +23,7 @@ func defaultConfig() conf { transport.Timeout = 30 * time.Second return conf{ + EnrichWith: []string{"groups"}, SyncInterval: 24 * time.Hour, UpdateInterval: 15 * time.Minute, LimitWindow: time.Minute, @@ -48,6 +49,12 @@ type conf struct { // the API. It can be ""/"all", "users", or // "devices". Dataset string `config:"dataset"` + // EnrichWith specifies the additional data that + // will be used to enrich user data. It can include + // "groups", "roles" and "factors". + // If it is a single element with "none", no + // enrichment is performed. + EnrichWith []string `config:"enrich_with"` // SyncInterval is the time between full // synchronisation operations. diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 404f069a7ec2..141e6ac6aea4 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -15,6 +15,7 @@ import ( "net/url" "os" "path/filepath" + "slices" "strings" "time" @@ -460,7 +461,7 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn if fullSync { for _, u := range batch { - p.addGroup(ctx, u, state) + p.addUserMetadata(ctx, u, state) if u.LastUpdated.After(lastUpdated) { lastUpdated = u.LastUpdated } @@ -468,7 +469,7 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn } else { users = grow(users, len(batch)) for _, u := range batch { - su := p.addGroup(ctx, u, state) + su := p.addUserMetadata(ctx, u, state) users = append(users, su) if u.LastUpdated.After(lastUpdated) { lastUpdated = u.LastUpdated @@ -500,14 +501,41 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn return users, nil } -func (p *oktaInput) addGroup(ctx context.Context, u okta.User, state *stateStore) *User { +func (p *oktaInput) addUserMetadata(ctx context.Context, u okta.User, state *stateStore) *User { su := state.storeUser(u) - groups, _, err := okta.GetUserGroupDetails(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger) - if err != nil { - p.logger.Warnf("failed to get user group membership for %s: %v", u.ID, err) + switch len(p.cfg.EnrichWith) { + case 1: + if p.cfg.EnrichWith[0] != "none" { + break + } + fallthrough + case 0: return su } - su.Groups = groups + if slices.Contains(p.cfg.EnrichWith, "groups") { + groups, _, err := okta.GetUserGroupDetails(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger) + if err != nil { + p.logger.Warnf("failed to get user group membership for %s: %v", u.ID, err) + } else { + su.Groups = groups + } + } + if slices.Contains(p.cfg.EnrichWith, "factors") { + factors, _, err := okta.GetUserFactors(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger) + if err != nil { + p.logger.Warnf("failed to get user factors for %s: %v", u.ID, err) + } else { + su.Factors = factors + } + } + if slices.Contains(p.cfg.EnrichWith, "roles") { + roles, _, err := okta.GetUserRoles(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, u.ID, p.lim, p.cfg.LimitWindow, p.logger) + if err != nil { + p.logger.Warnf("failed to get user roles for %s: %v", u.ID, err) + } else { + su.Roles = append(su.Roles, roles...) + } + } return su } diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go index c13cf4040c5f..5752370c4ce2 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "net/url" "path" + "slices" "strings" "testing" "time" @@ -31,13 +32,14 @@ func TestOktaDoFetch(t *testing.T) { tests := []struct { dataset string + enrichWith []string wantUsers bool wantDevices bool }{ - {dataset: "", wantUsers: true, wantDevices: true}, - {dataset: "all", wantUsers: true, wantDevices: true}, - {dataset: "users", wantUsers: true, wantDevices: false}, - {dataset: "devices", wantUsers: false, wantDevices: true}, + {dataset: "", enrichWith: []string{"groups"}, wantUsers: true, wantDevices: true}, + {dataset: "all", enrichWith: []string{"groups"}, wantUsers: true, wantDevices: true}, + {dataset: "users", enrichWith: []string{"groups", "roles", "factors"}, wantUsers: true, wantDevices: false}, + {dataset: "devices", enrichWith: []string{"groups"}, wantUsers: false, wantDevices: true}, } for _, test := range tests { @@ -56,14 +58,18 @@ func TestOktaDoFetch(t *testing.T) { window = time.Minute key = "token" users = `[{"id":"USERID","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/USERID"}}}]` + roles = `[{"id":"IFIFAX2BIRGUSTQ","label":"Application administrator","type":"APP_ADMIN","status":"ACTIVE","created":"2019-02-06T16:17:40.000Z","lastUpdated":"2019-02-06T16:17:40.000Z","assignmentType":"USER"},{"id":"JBCUYUC7IRCVGS27IFCE2SKO","label":"Help Desk administrator","type":"HELP_DESK_ADMIN","status":"ACTIVE","created":"2019-02-06T16:17:40.000Z","lastUpdated":"2019-02-06T16:17:40.000Z","assignmentType":"USER"},{"id":"ra125eqBFpETrMwu80g4","label":"Organization administrator","type":"ORG_ADMIN","status":"ACTIVE","created":"2019-02-06T16:17:40.000Z","lastUpdated":"2019-02-06T16:17:40.000Z","assignmentType":"USER"},{"id":"gra25fapn1prGTBKV0g4","label":"API Access Management administrator","type":"API_ACCESS_MANAGEMENT_ADMIN","status":"ACTIVE","created\"":"2019-02-06T16:20:57.000Z","lastUpdated\"":"2019-02-06T16:20:57.000Z","assignmentType\"":"GROUP"}]` groups = `[{"id":"USERID","profile":{"description":"All users in your organization","name":"Everyone"}}]` + factors = `[{"id":"ufs2bysphxKODSZKWVCT","factorType":"question","provider":"OKTA","vendorName":"OKTA","status":"ACTIVE","created":"2014-04-15T18:10:06.000Z","lastUpdated":"2014-04-15T18:10:06.000Z","profile":{"question":"favorite_art_piece","questionText":"What is your favorite piece of art?"}},{"id":"ostf2gsyictRQDSGTDZE","factorType":"token:software:totp","provider":"OKTA","status":"PENDING_ACTIVATION","created":"2014-06-27T20:27:33.000Z","lastUpdated":"2014-06-27T20:27:33.000Z","profile":{"credentialId":"dade.murphy@example.com"}},{"id":"sms2gt8gzgEBPUWBIFHN","factorType":"sms","provider":"OKTA","status":"ACTIVE","created":"2014-06-27T20:27:26.000Z","lastUpdated":"2014-06-27T20:27:26.000Z","profile":{"phoneNumber":"+1-555-415-1337"}}]` devices = `[{"id":"DEVICEID","status":"STATUS","created":"2019-10-02T18:03:07.000Z","lastUpdated":"2019-10-02T18:03:07.000Z","profile":{"displayName":"Example Device name 1","platform":"WINDOWS","serialNumber":"XXDDRFCFRGF3M8MD6D","sid":"S-1-11-111","registered":true,"secureHardwarePresent":false,"diskEncryptionType":"ALL_INTERNAL_VOLUMES"},"resourceType":"UDDevice","resourceDisplayName":{"value":"Example Device name 1","sensitive":false},"resourceAlternateId":null,"resourceId":"DEVICEID","_links":{"activate":{"href":"https://localhost/api/v1/devices/DEVICEID/lifecycle/activate","hints":{"allow":["POST"]}},"self":{"href":"https://localhost/api/v1/devices/DEVICEID","hints":{"allow":["GET","PATCH","PUT"]}},"users":{"href":"https://localhost/api/v1/devices/DEVICEID/users","hints":{"allow":["GET"]}}}}]` ) data := map[string]string{ "users": users, + "roles": roles, "groups": groups, "devices": devices, + "factors": factors, } var wantUsers []User @@ -88,29 +94,50 @@ func TestOktaDoFetch(t *testing.T) { t.Fatalf("failed to unmarshal device data: %v", err) } } + var wantFactors []okta.Factor + if slices.Contains(test.enrichWith, "factors") { + err := json.Unmarshal([]byte(factors), &wantFactors) + if err != nil { + t.Fatalf("failed to unmarshal factor data: %v", err) + } + } + var wantRoles []okta.Role + if slices.Contains(test.enrichWith, "roles") { + err := json.Unmarshal([]byte(roles), &wantRoles) + if err != nil { + t.Fatalf("failed to unmarshal role data: %v", err) + } + } wantStates := make(map[string]State) // Set the number of repeats. const repeats = 3 var n int - ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + setHeaders := func(w http.ResponseWriter) { // Leave 49 remaining, reset in one minute. w.Header().Add("x-rate-limit-limit", "50") w.Header().Add("x-rate-limit-remaining", "49") w.Header().Add("x-rate-limit-reset", fmt.Sprint(time.Now().Add(time.Minute).Unix())) - - if strings.HasPrefix(r.URL.Path, "/api/v1/users") && strings.HasSuffix(r.URL.Path, "groups") { - // Give the groups if this is a get user groups request. - userid := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/users/"), "/groups") - fmt.Fprintln(w, strings.ReplaceAll(data["groups"], "USERID", userid)) - return - } - if strings.HasPrefix(r.URL.Path, "/api/v1/device") && strings.HasSuffix(r.URL.Path, "users") { - // Give one user if this is a get device users request. - fmt.Fprintln(w, data["users"]) + } + mux := http.NewServeMux() + mux.Handle("/api/v1/users/{userid}/{metadata}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + setHeaders(w) + attr := r.PathValue("metadata") + if attr != "groups" { + fmt.Fprintln(w, data[attr]) return } + // Give the groups if this is a get user groups request. + userid := r.PathValue("userid") + fmt.Fprintln(w, strings.ReplaceAll(data[attr], "USERID", userid)) + })) + mux.Handle("/api/v1/devices/{deviceid}/users", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + setHeaders(w) + fmt.Fprintln(w, data["users"]) + })) + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + setHeaders(w) base := path.Base(r.URL.Path) @@ -143,6 +170,7 @@ func TestOktaDoFetch(t *testing.T) { ) fmt.Fprintln(w, replacer.Replace(data[base])) })) + ts := httptest.NewTLSServer(mux) defer ts.Close() u, err := url.Parse(ts.URL) @@ -154,6 +182,7 @@ func TestOktaDoFetch(t *testing.T) { OktaDomain: u.Host, OktaToken: key, Dataset: test.dataset, + EnrichWith: test.enrichWith, }, client: ts.Client(), lim: rate.NewLimiter(1, 1), @@ -196,6 +225,12 @@ func TestOktaDoFetch(t *testing.T) { if g.ID != wantID { t.Errorf("unexpected user ID for user %d: got:%s want:%s", i, g.ID, wantID) } + if len(g.Factors) != len(wantFactors) { + t.Errorf("number of factors for user %d: got:%d want:%d", i, len(g.Factors), len(wantFactors)) + } + if len(g.Roles) != len(wantRoles) { + t.Errorf("number of roles for user %d: got:%d want:%d", i, len(g.Roles), len(wantRoles)) + } for j, gg := range g.Groups { if gg.ID != wantID { t.Errorf("unexpected used ID for user group %d in %d: got:%s want:%s", j, i, gg.ID, wantID) diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go b/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go index 401b3353d14a..4e9254e56b84 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go @@ -37,8 +37,10 @@ const ( type User struct { okta.User `json:"properties"` - Groups []okta.Group `json:"groups"` - State State `json:"state"` + Groups []okta.Group `json:"groups"` + Roles []okta.Role `json:"roles"` + Factors []okta.Factor `json:"factors"` + State State `json:"state"` } type Device struct { From 34c79d87a464d6dad8a03171e5e65f52ea573ace Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 5 Nov 2024 08:22:57 +1030 Subject: [PATCH 2/2] address pr comment --- x-pack/filebeat/input/entityanalytics/provider/okta/okta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 141e6ac6aea4..5d68cf3f5c4a 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -533,7 +533,7 @@ func (p *oktaInput) addUserMetadata(ctx context.Context, u okta.User, state *sta if err != nil { p.logger.Warnf("failed to get user roles for %s: %v", u.ID, err) } else { - su.Roles = append(su.Roles, roles...) + su.Roles = roles } } return su