From 930aa1ecb45b3dad059666299aa45c206c2ed7f3 Mon Sep 17 00:00:00 2001 From: Mia Epner Date: Mon, 3 Apr 2023 12:21:15 +0200 Subject: [PATCH] create helper types and functions for the summary namespace and month maps --- vault/activity_log.go | 135 ++++++++++++++++++++----------------- vault/activity_log_test.go | 59 ++++++++++++++++ 2 files changed, 131 insertions(+), 63 deletions(-) diff --git a/vault/activity_log.go b/vault/activity_log.go index 00e685b9e524..aad726d34c6e 100644 --- a/vault/activity_log.go +++ b/vault/activity_log.go @@ -1872,6 +1872,12 @@ func (a *ActivityLog) namespaceToLabel(ctx context.Context, nsID string) string return ns.Path } +type ( + summaryByNamespace map[string]*processByNamespace + summaryByMount map[string]*processMount + summaryByMonth map[int64]*processMonth +) + type processCounts struct { // entityID -> present Entities map[string]struct{} @@ -1889,6 +1895,23 @@ func newProcessCounts() *processCounts { } } +func (p *processCounts) add(client *activity.EntityRecord) { + if client.NonEntity { + p.NonEntities[client.ClientID] = struct{}{} + } else { + p.Entities[client.ClientID] = struct{}{} + } +} + +func (p *processCounts) contains(client *activity.EntityRecord) bool { + if client.NonEntity { + _, ok := p.NonEntities[client.ClientID] + return ok + } + _, ok := p.Entities[client.ClientID] + return ok +} + type processMount struct { Counts *processCounts } @@ -1899,105 +1922,91 @@ func newProcessMount() *processMount { } } +func (p *processMount) add(client *activity.EntityRecord) { + p.Counts.add(client) +} + +func (s summaryByMount) add(client *activity.EntityRecord) { + if _, present := s[client.MountAccessor]; !present { + s[client.MountAccessor] = newProcessMount() + } + s[client.MountAccessor].add(client) +} + type processByNamespace struct { Counts *processCounts - Mounts map[string]*processMount + Mounts summaryByMount } func newByNamespace() *processByNamespace { return &processByNamespace{ Counts: newProcessCounts(), - Mounts: make(map[string]*processMount), + Mounts: make(summaryByMount), } } +func (p *processByNamespace) add(client *activity.EntityRecord) { + p.Counts.add(client) + p.Mounts.add(client) +} + +func (s summaryByNamespace) add(client *activity.EntityRecord) { + if _, present := s[client.NamespaceID]; !present { + s[client.NamespaceID] = newByNamespace() + } + s[client.NamespaceID].add(client) +} + type processNewClients struct { Counts *processCounts - Namespaces map[string]*processByNamespace + Namespaces summaryByNamespace } func newProcessNewClients() *processNewClients { return &processNewClients{ Counts: newProcessCounts(), - Namespaces: make(map[string]*processByNamespace), + Namespaces: make(summaryByNamespace), } } +func (p *processNewClients) add(client *activity.EntityRecord) { + p.Counts.add(client) + p.Namespaces.add(client) +} + type processMonth struct { Counts *processCounts - Namespaces map[string]*processByNamespace + Namespaces summaryByNamespace NewClients *processNewClients } func newProcessMonth() *processMonth { return &processMonth{ Counts: newProcessCounts(), - Namespaces: make(map[string]*processByNamespace), + Namespaces: make(summaryByNamespace), NewClients: newProcessNewClients(), } } -// processClientRecord parses the client record e and stores the breakdowns in -// the maps provided. -func processClientRecord(e *activity.EntityRecord, byNamespace map[string]*processByNamespace, byMonth map[int64]*processMonth, startTime time.Time) { - if _, present := byNamespace[e.NamespaceID]; !present { - byNamespace[e.NamespaceID] = newByNamespace() - } - - if _, present := byNamespace[e.NamespaceID].Mounts[e.MountAccessor]; !present { - byNamespace[e.NamespaceID].Mounts[e.MountAccessor] = newProcessMount() - } - - if e.NonEntity { - byNamespace[e.NamespaceID].Counts.NonEntities[e.ClientID] = struct{}{} - byNamespace[e.NamespaceID].Mounts[e.MountAccessor].Counts.NonEntities[e.ClientID] = struct{}{} - } else { - byNamespace[e.NamespaceID].Counts.Entities[e.ClientID] = struct{}{} - byNamespace[e.NamespaceID].Mounts[e.MountAccessor].Counts.Entities[e.ClientID] = struct{}{} - } +func (p *processMonth) add(client *activity.EntityRecord) { + p.Counts.add(client) + p.NewClients.add(client) + p.Namespaces.add(client) +} +func (s summaryByMonth) add(client *activity.EntityRecord, startTime time.Time) { monthTimestamp := timeutil.StartOfMonth(startTime).UTC().Unix() - if _, present := byMonth[monthTimestamp]; !present { - byMonth[monthTimestamp] = newProcessMonth() - } - - if _, present := byMonth[monthTimestamp].Namespaces[e.NamespaceID]; !present { - byMonth[monthTimestamp].Namespaces[e.NamespaceID] = newByNamespace() - } - - if _, present := byMonth[monthTimestamp].Namespaces[e.NamespaceID].Mounts[e.MountAccessor]; !present { - byMonth[monthTimestamp].Namespaces[e.NamespaceID].Mounts[e.MountAccessor] = newProcessMount() + if _, present := s[monthTimestamp]; !present { + s[monthTimestamp] = newProcessMonth() } + s[monthTimestamp].add(client) +} - if _, present := byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID]; !present { - byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID] = newByNamespace() - } - - if _, present := byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID].Mounts[e.MountAccessor]; !present { - byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID].Mounts[e.MountAccessor] = newProcessMount() - } - - // At first assume all the clients in the given month, as new. - // Before persisting this information to disk, clients that have - // activity in the previous months of a given billing cycle will be - // deleted. - if e.NonEntity == true { - byMonth[monthTimestamp].Counts.NonEntities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].Namespaces[e.NamespaceID].Counts.NonEntities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].Namespaces[e.NamespaceID].Mounts[e.MountAccessor].Counts.NonEntities[e.ClientID] = struct{}{} - - byMonth[monthTimestamp].NewClients.Counts.NonEntities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID].Counts.NonEntities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID].Mounts[e.MountAccessor].Counts.NonEntities[e.ClientID] = struct{}{} - } else { - byMonth[monthTimestamp].Counts.Entities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].Namespaces[e.NamespaceID].Counts.Entities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].Namespaces[e.NamespaceID].Mounts[e.MountAccessor].Counts.Entities[e.ClientID] = struct{}{} - - byMonth[monthTimestamp].NewClients.Counts.Entities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID].Counts.Entities[e.ClientID] = struct{}{} - byMonth[monthTimestamp].NewClients.Namespaces[e.NamespaceID].Mounts[e.MountAccessor].Counts.Entities[e.ClientID] = struct{}{} - } +// processClientRecord parses the client record e and stores the breakdowns in +// the maps provided. +func processClientRecord(e *activity.EntityRecord, byNamespace summaryByNamespace, byMonth summaryByMonth, startTime time.Time) { + byNamespace.add(e) + byMonth.add(e, startTime) } // goroutine to process the request in the intent log, creating precomputed queries. diff --git a/vault/activity_log_test.go b/vault/activity_log_test.go index afa7fd469dc8..8f29113cd41d 100644 --- a/vault/activity_log_test.go +++ b/vault/activity_log_test.go @@ -4233,3 +4233,62 @@ func TestActivityLog_partialMonthClientCountWithMultipleMountPaths(t *testing.T) } } } + +// TestActivityLog_processClientRecord calls processClientRecord for an entity and a non-entity record and verifies that +// the record is present in the namespace and month maps +func TestActivityLog_processClientRecord(t *testing.T) { + startTime := time.Now() + mount := "mount" + namespace := "namespace" + clientID := "client-id" + run := func(t *testing.T, isNonEntity bool) { + t.Helper() + record := &activity.EntityRecord{ + MountAccessor: mount, + NamespaceID: namespace, + ClientID: clientID, + NonEntity: isNonEntity, + } + byNS := make(summaryByNamespace) + byMonth := make(summaryByMonth) + processClientRecord(record, byNS, byMonth, startTime) + require.Contains(t, byNS, namespace) + require.Contains(t, byNS[namespace].Mounts, mount) + monthIndex := timeutil.StartOfMonth(startTime).UTC().Unix() + require.Contains(t, byMonth, monthIndex) + require.Equal(t, byMonth[monthIndex].Namespaces, byNS) + require.Equal(t, byMonth[monthIndex].NewClients.Namespaces, byNS) + + if isNonEntity { + require.Contains(t, byMonth[monthIndex].Counts.NonEntities, clientID) + require.NotContains(t, byMonth[monthIndex].Counts.Entities, clientID) + + require.Contains(t, byMonth[monthIndex].NewClients.Counts.NonEntities, clientID) + require.NotContains(t, byMonth[monthIndex].NewClients.Counts.Entities, clientID) + + require.Contains(t, byNS[namespace].Mounts[mount].Counts.NonEntities, clientID) + require.Contains(t, byNS[namespace].Counts.NonEntities, clientID) + + require.NotContains(t, byNS[namespace].Mounts[mount].Counts.Entities, clientID) + require.NotContains(t, byNS[namespace].Counts.Entities, clientID) + } else { + require.Contains(t, byMonth[monthIndex].Counts.Entities, clientID) + require.NotContains(t, byMonth[monthIndex].Counts.NonEntities, clientID) + + require.Contains(t, byMonth[monthIndex].NewClients.Counts.Entities, clientID) + require.NotContains(t, byMonth[monthIndex].NewClients.Counts.NonEntities, clientID) + + require.Contains(t, byNS[namespace].Mounts[mount].Counts.Entities, clientID) + require.Contains(t, byNS[namespace].Counts.Entities, clientID) + + require.NotContains(t, byNS[namespace].Mounts[mount].Counts.NonEntities, clientID) + require.NotContains(t, byNS[namespace].Counts.NonEntities, clientID) + } + } + t.Run("non entity", func(t *testing.T) { + run(t, true) + }) + t.Run("entity", func(t *testing.T) { + run(t, false) + }) +}