From e8d614120cf18d89ed222032b087836e03461d7a Mon Sep 17 00:00:00 2001 From: Jumpy Squirrel Date: Tue, 17 Dec 2024 16:32:06 +0100 Subject: [PATCH 1/3] feat(#240): cache avatar in attendee --- docs/config-template.yaml | 1 + internal/entity/attendee.go | 1 + internal/repository/config/structure.go | 1 + internal/service/attendeesrv/attendeesrv.go | 1 + internal/service/attendeesrv/dues.go | 20 +++++++++++++++++--- internal/web/middleware/security.go | 2 ++ internal/web/util/ctxvalues/accessors.go | 9 +++++++++ 7 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/config-template.yaml b/docs/config-template.yaml index 2eb432c..7c419b3 100644 --- a/docs/config-template.yaml +++ b/docs/config-template.yaml @@ -45,6 +45,7 @@ security: audience: 'only-allowed-audience-in-tokens' # optional, but will be checked if set issuer: 'only-allowed-issuer-in-tokens' + avatar_base_url: 'http://localhost:10000/identity-avatars/' cors: # set this to true to send disable cors headers - not for production - local/test instances only - will log lots of warnings disable: false diff --git a/internal/entity/attendee.go b/internal/entity/attendee.go index 75341f1..1353239 100644 --- a/internal/entity/attendee.go +++ b/internal/entity/attendee.go @@ -32,6 +32,7 @@ type Attendee struct { Options string `gorm:"type:varchar(4096) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"` // comma-separated choice field with leading and trailing comma UserComments string `gorm:"type:text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" testdiff:"ignore"` Identity string `gorm:"type:varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;uniqueIndex:att_attendees_identity_uidx"` + Avatar string `gorm:"type:varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"` CacheTotalDues int64 `testdiff:"ignore"` // cache for search functionality only: valid dues balance CachePaymentBalance int64 `testdiff:"ignore"` // cache for search functionality only: valid payments balance CacheOpenBalance int64 `testdiff:"ignore"` // cache for search functionality only: tentative + pending payments balance diff --git a/internal/repository/config/structure.go b/internal/repository/config/structure.go index 9be64fd..337f75d 100644 --- a/internal/repository/config/structure.go +++ b/internal/repository/config/structure.go @@ -91,6 +91,7 @@ type ( EarlyRegGroup string `yaml:"early_reg_group"` // optional, the group claim that turns on early registration Audience string `yaml:"audience"` Issuer string `yaml:"issuer"` + AvatarBaseUrl string `yaml:"avatar_base_url"` // optional, prefix for the avatar from the JWT id token } CorsConfig struct { diff --git a/internal/service/attendeesrv/attendeesrv.go b/internal/service/attendeesrv/attendeesrv.go index 5461695..9b50221 100644 --- a/internal/service/attendeesrv/attendeesrv.go +++ b/internal/service/attendeesrv/attendeesrv.go @@ -37,6 +37,7 @@ func (s *AttendeeServiceImplData) RegisterNewAttendee(ctx context.Context, atten } else { // record which user owns this attendee attendee.Identity = ctxvalues.Subject(ctx) + attendee.Avatar = ctxvalues.Avatar(ctx) // if an admin registers a guest, they'll get the admin's avatar, but that's "correct", as the admin is now also the owner of the registration } if config.RequireLoginForReg() { diff --git a/internal/service/attendeesrv/dues.go b/internal/service/attendeesrv/dues.go index 7f12b3b..fd8d64c 100644 --- a/internal/service/attendeesrv/dues.go +++ b/internal/service/attendeesrv/dues.go @@ -12,6 +12,7 @@ import ( "github.com/eurofurence/reg-attendee-service/internal/repository/config" "github.com/eurofurence/reg-attendee-service/internal/repository/database" "github.com/eurofurence/reg-attendee-service/internal/repository/paymentservice" + "github.com/eurofurence/reg-attendee-service/internal/web/util/ctxvalues" "strconv" "strings" ) @@ -207,13 +208,15 @@ func (s *AttendeeServiceImplData) UpdateAttendeeCacheAndCalculateResultingStatus identity := s.suffixForDeletedAttendees(attendee, newStatus, attendee.Identity) zip := s.suffixForDeletedAttendees(attendee, newStatus, attendee.Zip) + avatar := s.avatarIfMatchingUser(ctx, identity) + dues, payments, open, dueDate := s.balances(updatedTransactionHistory) // never move due date back in time (allows manual override) if attendee.CacheDueDate != "" && attendee.CacheDueDate > dueDate { dueDate = attendee.CacheDueDate } - duesInformationChanged, err := s.updateCachedValuesAndIdentityInAttendee(ctx, attendee, dues, payments, open, dueDate, identity, zip) + duesInformationChanged, err := s.updateCachedValuesAndIdentityInAttendee(ctx, attendee, dues, payments, open, dueDate, identity, avatar, zip) if err != nil { return newStatus, false, err } @@ -239,7 +242,14 @@ func (s *AttendeeServiceImplData) suffixForDeletedAttendees(attendee *entity.Att return value } -func (s *AttendeeServiceImplData) updateCachedValuesAndIdentityInAttendee(ctx context.Context, attendee *entity.Attendee, dues int64, payments int64, open int64, dueDate string, identity string, zip string) (bool, error) { +func (s *AttendeeServiceImplData) avatarIfMatchingUser(ctx context.Context, identity string) string { + if identity == ctxvalues.Subject(ctx) { + return ctxvalues.Avatar(ctx) + } + return "" +} + +func (s *AttendeeServiceImplData) updateCachedValuesAndIdentityInAttendee(ctx context.Context, attendee *entity.Attendee, dues int64, payments int64, open int64, dueDate string, identity string, avatar string, zip string) (bool, error) { duesRelevantUpdate := attendee.CacheTotalDues != dues || attendee.CachePaymentBalance != payments || attendee.CacheDueDate != dueDate @@ -247,7 +257,8 @@ func (s *AttendeeServiceImplData) updateCachedValuesAndIdentityInAttendee(ctx co needsUpdate := duesRelevantUpdate || attendee.CacheOpenBalance != open || attendee.Identity != identity || - attendee.Zip != zip + attendee.Zip != zip || + (avatar != "" && attendee.Avatar != avatar) if needsUpdate { attendee.CacheTotalDues = dues @@ -256,6 +267,9 @@ func (s *AttendeeServiceImplData) updateCachedValuesAndIdentityInAttendee(ctx co attendee.CacheDueDate = dueDate attendee.Identity = identity attendee.Zip = zip + if avatar != "" { + attendee.Avatar = avatar + } err := database.GetRepository().UpdateAttendee(ctx, attendee) return duesRelevantUpdate, err } diff --git a/internal/web/middleware/security.go b/internal/web/middleware/security.go index ca489b9..e6575c4 100644 --- a/internal/web/middleware/security.go +++ b/internal/web/middleware/security.go @@ -118,6 +118,7 @@ type CustomClaims struct { EmailVerified bool `json:"email_verified"` Groups []string `json:"groups,omitempty"` Name string `json:"name"` + Avatar string `json:"avatar"` } type AllClaims struct { @@ -153,6 +154,7 @@ func checkIdToken_MustReturnOnError(ctx context.Context, idTokenValue string) (s ctxvalues.SetEmailVerified(ctx, parsedClaims.EmailVerified) ctxvalues.SetName(ctx, parsedClaims.Name) ctxvalues.SetSubject(ctx, parsedClaims.Subject) + ctxvalues.SetAvatar(ctx, parsedClaims.Avatar) for _, group := range parsedClaims.Groups { ctxvalues.SetAuthorizedAsGroup(ctx, group) } diff --git a/internal/web/util/ctxvalues/accessors.go b/internal/web/util/ctxvalues/accessors.go index 15895bd..1e537a0 100644 --- a/internal/web/util/ctxvalues/accessors.go +++ b/internal/web/util/ctxvalues/accessors.go @@ -18,6 +18,7 @@ const ContextEmail = "email" const ContextEmailVerified = "emailverified" const ContextName = "name" const ContextSubject = "subject" +const ContextAvatar = "avatar" func CreateContextWithValueMap(ctx context.Context) context.Context { // this is so we can add values to our context, like ... I don't know ... the http status from the response! @@ -108,6 +109,14 @@ func SetSubject(ctx context.Context, Subject string) { setValue(ctx, ContextSubject, Subject) } +func Avatar(ctx context.Context) string { + return valueOrDefault(ctx, ContextAvatar, "") +} + +func SetAvatar(ctx context.Context, avatar string) { + setValue(ctx, ContextAvatar, avatar) +} + func HasApiToken(ctx context.Context) bool { v := valueOrDefault(ctx, ContextApiToken, "") return v == config.FixedApiToken() From c9bba4ed9bc91fc99b63e42b1d3697035fe8815a Mon Sep 17 00:00:00 2001 From: Jumpy Squirrel Date: Tue, 17 Dec 2024 16:52:56 +0100 Subject: [PATCH 2/3] feat(#246): return identity_subject and avatar in search results if selected --- api/openapi-spec/openapi.yaml | 8 ++++++++ internal/api/v1/attendee/api.go | 2 ++ internal/repository/config/config.go | 4 ++++ internal/service/attendeesrv/search.go | 13 ++++++++++++- internal/web/controller/adminctl/adminctl.go | 2 +- test/acceptance/admin_acc_test.go | 9 ++++++--- 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/api/openapi-spec/openapi.yaml b/api/openapi-spec/openapi.yaml index 8bf6988..dd17ca9 100644 --- a/api/openapi-spec/openapi.yaml +++ b/api/openapi-spec/openapi.yaml @@ -2043,6 +2043,12 @@ components: admin_comments: type: string description: Optional comments only visible to admins. Not processed in any way. + identity_subject: + type: string + description: the OIDC subject the registration was made or last updated with. + avatar: + type: string + description: the avatar URL from when the registration was last updated by the user in the registration system. May be empty or sometimes outdated. AttendeeSearchCriteria: type: object required: @@ -2313,6 +2319,8 @@ components: - due_date - registered - admin_comments + - identity_subject + - avatar # and the field sets - name - address diff --git a/internal/api/v1/attendee/api.go b/internal/api/v1/attendee/api.go index c0f55c8..d6eed61 100644 --- a/internal/api/v1/attendee/api.go +++ b/internal/api/v1/attendee/api.go @@ -128,6 +128,8 @@ type AttendeeSearchResult struct { DueDate *string `json:"due_date,omitempty"` Registered *string `json:"registered,omitempty"` AdminComments *string `json:"admin_comments,omitempty"` + IdentitySubject *string `json:"identity_subject"` + Avatar *string `json:"avatar"` } // --- flags/options/packages result --- diff --git a/internal/repository/config/config.go b/internal/repository/config/config.go index 081f0c5..782ab6d 100644 --- a/internal/repository/config/config.go +++ b/internal/repository/config/config.go @@ -337,3 +337,7 @@ func VatPercent() float64 { func RegsysPublicUrl() string { return Configuration().Service.RegsysPublicUrl } + +func AvatarBaseUrl() string { + return Configuration().Security.Oidc.AvatarBaseUrl +} diff --git a/internal/service/attendeesrv/search.go b/internal/service/attendeesrv/search.go index 7e31f4c..76c5192 100644 --- a/internal/service/attendeesrv/search.go +++ b/internal/service/attendeesrv/search.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/eurofurence/reg-attendee-service/internal/api/v1/attendee" + "github.com/eurofurence/reg-attendee-service/internal/api/v1/status" "github.com/eurofurence/reg-attendee-service/internal/entity" "github.com/eurofurence/reg-attendee-service/internal/repository/config" "github.com/eurofurence/reg-attendee-service/internal/repository/database" @@ -31,7 +32,7 @@ func (s *AttendeeServiceImplData) mapToAttendeeSearchResult(att *entity.Attendee if len(fillFields) == 0 { fillFields = []string{"nickname", "name", "country", "spoken_languages", "email", "telegram", "birthday", "pronouns", "tshirt_size", "flags", "options", "packages", "user_comments", "status", - "total_dues", "payment_balance", "current_dues", "due_date", "registered", "admin_comments"} + "total_dues", "payment_balance", "current_dues", "due_date", "registered", "admin_comments", "avatar"} } var currentDues = att.CacheTotalDues - att.CachePaymentBalance @@ -41,6 +42,14 @@ func (s *AttendeeServiceImplData) mapToAttendeeSearchResult(att *entity.Attendee options := removeWrappingCommas(att.Options) packagesList := sortedPackageListFromCommaSeparatedWithCounts(removeWrappingCommas(att.Packages)) packages := packagesFromPackagesList(packagesList) + identity := "" + if att.Status != status.Deleted { + identity = att.Identity + } + avatar := att.Avatar + if avatar != "" { + avatar = config.AvatarBaseUrl() + avatar + } return attendee.AttendeeSearchResult{ Id: att.ID, BadgeId: s.badgeId(att.ID), @@ -77,6 +86,8 @@ func (s *AttendeeServiceImplData) mapToAttendeeSearchResult(att *entity.Attendee DueDate: contains(n(att.CacheDueDate), fillFields, "all", "balances", "due_date"), Registered: contains(n(registered), fillFields, "all", "registered"), AdminComments: contains(n(att.AdminComments), fillFields, "all", "admin_comments"), + IdentitySubject: contains(n(identity), fillFields, "all", "identity_subject"), + Avatar: contains(n(avatar), fillFields, "all", "avatar"), } } diff --git a/internal/web/controller/adminctl/adminctl.go b/internal/web/controller/adminctl/adminctl.go index 53d347e..a04c8dd 100644 --- a/internal/web/controller/adminctl/adminctl.go +++ b/internal/web/controller/adminctl/adminctl.go @@ -141,7 +141,7 @@ func limitToAllowedFields(desired []string) []string { allowed := []string{"id", "nickname", "first_name", "last_name", "country", "spoken_languages", "registration_language", "birthday", "pronouns", "tshirt_size", "flags", "options", "packages", "status", - "total_dues", "payment_balance", "current_dues", + "total_dues", "payment_balance", "current_dues", "identity_subject", "avatar", } result := make([]string, 0) diff --git a/test/acceptance/admin_acc_test.go b/test/acceptance/admin_acc_test.go index 706d6bc..7b1049f 100644 --- a/test/acceptance/admin_acc_test.go +++ b/test/acceptance/admin_acc_test.go @@ -1121,7 +1121,8 @@ func TestSearch_RegdeskOk(t *testing.T) { "status": "approved", "total_dues": 25500, "payment_balance": 0, - "current_dues": 25500 + "current_dues": 25500, + "identity_subject": "1234567890" } ] }` @@ -1194,7 +1195,8 @@ func TestSearch_SponsordeskOk(t *testing.T) { "status": "paid", "total_dues": 25500, "payment_balance": 25500, - "current_dues": 0 + "current_dues": 0, + "identity_subject": "1234567890" } ] }` @@ -1396,7 +1398,8 @@ func TestSearch_AdminOk(t *testing.T) { "status": "new", "total_dues": 0, "payment_balance": 0, - "current_dues": 0 + "current_dues": 0, + "identity_subject": "1234567890" } ] }` From f30c8362850bb254afdd0d2c52c171b15a62dc44 Mon Sep 17 00:00:00 2001 From: Jumpy Squirrel Date: Tue, 17 Dec 2024 17:57:42 +0100 Subject: [PATCH 3/3] feat(#246): allow searching for a small list of identity_subjects --- api/openapi-spec/openapi.yaml | 15 ++++++++++++++ internal/api/v1/attendee/api.go | 1 + .../repository/database/inmemorydb/match.go | 11 +++++++++- .../database/mysqldb/searchquery.go | 20 +++++++++++++++++++ .../database/mysqldb/searchquery_test.go | 8 ++++++-- 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/api/openapi-spec/openapi.yaml b/api/openapi-spec/openapi.yaml index dd17ca9..2ab22fe 100644 --- a/api/openapi-spec/openapi.yaml +++ b/api/openapi-spec/openapi.yaml @@ -2231,6 +2231,21 @@ components: sponsor-items: 1 key-hand-out: 0 overdue: 1 + identity_subjects: + description: |- + exact match to search for a set of OIDC subjects as cached in registrations. No condition if left empty. + + IMPORTANT: Use of this field is limited to 8 identities, any additional identities in you list will be silently ignored! + If you need to check a large number of identities, please request the identity_subject search result field, + and filter on your end! + + Note that caching of identities is configuration dependent, if disabled, no registrations will ever match if this is set. + type: array + maxItems: 8 + items: + type: string + example: + - '1234567890' ChoiceStateCondition: type: integer format: int64 diff --git a/internal/api/v1/attendee/api.go b/internal/api/v1/attendee/api.go index d6eed61..f7148c3 100644 --- a/internal/api/v1/attendee/api.go +++ b/internal/api/v1/attendee/api.go @@ -84,6 +84,7 @@ type AttendeeSearchSingleCriterion struct { Permissions map[string]int8 `json:"permissions"` AdminComments string `json:"admin_comments"` AddInfo map[string]int8 `json:"add_info"` // can only search for presence of a value for each area, Note: special area 'overdue' + IdentitySubjects []string `json:"identity_subjects"` } // --- search result --- diff --git a/internal/repository/database/inmemorydb/match.go b/internal/repository/database/inmemorydb/match.go index 1b52a16..96cd2b4 100644 --- a/internal/repository/database/inmemorydb/match.go +++ b/internal/repository/database/inmemorydb/match.go @@ -8,6 +8,7 @@ import ( "github.com/eurofurence/reg-attendee-service/internal/repository/config" "github.com/eurofurence/reg-attendee-service/internal/web/util/validation" "github.com/ryanuber/go-glob" + "slices" "strconv" "strings" ) @@ -48,7 +49,8 @@ func (r *InMemoryRepository) matches(cond *attendee.AttendeeSearchSingleCriterio matchesSubstringGlobOrEmpty(cond.AdminComments, adm.AdminComments) && matchesAddInfoPresence(cond.AddInfo, addInf) && matchesOverdue(cond.AddInfo, a.CacheDueDate, r.Now().Format(config.IsoDateFormat), st.Status) && - matchesIsoDateRange(cond.BirthdayFrom, cond.BirthdayTo, a.Birthday) + matchesIsoDateRange(cond.BirthdayFrom, cond.BirthdayTo, a.Birthday) && + matchesIdentitySubjects(cond.IdentitySubjects, a.Identity) } func matchesUintSliceOrEmpty(cond []uint, value uint) bool { @@ -183,3 +185,10 @@ func matchesIsoDateRange(condFrom string, condTo string, value string) bool { } return true } + +func matchesIdentitySubjects(cond []string, value string) bool { + if len(cond) > 8 { + cond = cond[:8] + } + return len(cond) == 0 || slices.Contains(cond, value) +} diff --git a/internal/repository/database/mysqldb/searchquery.go b/internal/repository/database/mysqldb/searchquery.go index 215184d..3bb8952 100644 --- a/internal/repository/database/mysqldb/searchquery.go +++ b/internal/repository/database/mysqldb/searchquery.go @@ -261,6 +261,9 @@ func (r *MysqlRepository) addSingleCondition(cond *attendee.AttendeeSearchSingle if cond.BirthdayTo != "" { query.WriteString(stringLessThanOrEqual("a.birthday", cond.BirthdayTo, params, paramBaseName, ¶mNo)) } + if len(cond.IdentitySubjects) > 0 { + query.WriteString(stringSliceMatch("a.identity", cond.IdentitySubjects, params, paramBaseName, ¶mNo)) + } return query.String() } @@ -315,6 +318,23 @@ func safeStatusSliceMatch(field string, values []status.Status) string { return fmt.Sprintf(" AND ( %s IN (%s) )\n", field, strings.Join(mappedValues, ",")) } +func stringSliceMatch(field string, values []string, params map[string]interface{}, paramBaseName string, idx *int) string { + var result strings.Builder + result.WriteString(fmt.Sprintf(" AND ( %s IN ( ", field)) + cond := make([]string, 0) + for i, v := range values { + if i < 8 { + pName := fmt.Sprintf("%s_%d_%d", paramBaseName, *idx, i+1) + params[pName] = v + cond = append(cond, "@"+pName) + } + } + *idx++ + result.WriteString(strings.Join(cond, " , ")) + result.WriteString(" ) )\n") + return result.String() +} + func substringMatch(field string, condition string, params map[string]interface{}, paramBaseName string, idx *int) string { return fullstringMatch(field, "*"+condition+"*", params, paramBaseName, idx) } diff --git a/internal/repository/database/mysqldb/searchquery_test.go b/internal/repository/database/mysqldb/searchquery_test.go index 2673dbc..e37f1a3 100644 --- a/internal/repository/database/mysqldb/searchquery_test.go +++ b/internal/repository/database/mysqldb/searchquery_test.go @@ -110,8 +110,9 @@ func TestTwoFullSearchQueries(t *testing.T) { "overdue": 1, "sponsor-items": 1, }, - BirthdayFrom: "1970-10-24", - BirthdayTo: "1980-12-24", + BirthdayFrom: "1970-10-24", + BirthdayTo: "1980-12-24", + IdentitySubjects: []string{"Q1E4D2", "R1E5DD"}, }, }, MinId: 1, @@ -158,6 +159,8 @@ func TestTwoFullSearchQueries(t *testing.T) { "param_2_1": "small%bird", "param_2_2": "Johnny", "param_2_20": "1980-12-24", + "param_2_21_1": "Q1E4D2", + "param_2_21_2": "R1E5DD", "param_2_3": "%Berlin%", "param_2_4": "CH", "param_2_5": "%gg@hh%", @@ -238,6 +241,7 @@ WHERE ( AND ( ( SELECT COUNT(*) FROM att_additional_infos WHERE attendee_id = a.id AND area = @param_2_18_2 ) > 0 ) AND ( STRCMP(a.birthday,@param_2_19) >= 0 ) AND ( STRCMP(a.birthday,@param_2_20) <= 0 ) + AND ( a.identity IN ( @param_2_21_1 , @param_2_21_2 ) ) ) ) AND a.id >= @param_0_1 AND a.id <= @param_0_2 ORDER BY CONCAT(a.first_name, ' ', a.last_name) DESC`