Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix segments fragments loss #23781

Merged
merged 10 commits into from
Oct 25, 2023
3 changes: 3 additions & 0 deletions changelog/23781.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
core/activity: Fixes segments fragment loss due to exceeding entry record size limit
```
18 changes: 9 additions & 9 deletions vault/activity_log.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,18 @@ const (
activitySegmentWriteTimeout = 1 * time.Minute

// Number of client records to store per segment. Each ClientRecord may
// consume upto 99 bytes; rounding it to 100bytes. Considering the storage
// limit of 512KB per storage entry, we can roughly store 512KB/100bytes =
// 5241 clients; rounding down to 5000 clients.
activitySegmentClientCapacity = 5000
// consume upto 99 bytes; rounding it to 100bytes. This []byte undergo JSON marshalling
// before adding them in storage increasing the size by approximately 4/3 times. Considering the storage
// limit of 512KB per storage entry, we can roughly store 512KB/(100bytes * 4/3) yielding approximately 3820 records.
ActivitySegmentClientCapacity = 3820

// Maximum number of segments per month. This allows for 700K entities per
// month; 700K/5K. These limits are geared towards controlling the storage
// month; 700K/3820 (ActivitySegmentClientCapacity). These limits are geared towards controlling the storage
// implications of persisting activity logs. If we hit a scenario where the
// storage consequences are less important in comparison to the accuracy of
// the client activity, these limits can be further relaxed or even be
// removed.
activityLogMaxSegmentPerMonth = 140
activityLogMaxSegmentPerMonth = 184

// trackedTWESegmentPeriod is a time period of a little over a month, and represents
// the amount of time that needs to pass after a 1.9 or later upgrade to result in
Expand Down Expand Up @@ -351,7 +351,7 @@ func (a *ActivityLog) saveCurrentSegmentToStorageLocked(ctx context.Context, for
}

// Will all new entities fit? If not, roll over to a new segment.
available := activitySegmentClientCapacity - len(a.currentSegment.currentClients.Clients)
available := ActivitySegmentClientCapacity - len(a.currentSegment.currentClients.Clients)
remaining := available - len(newEntities)
excess := 0
if remaining < 0 {
Expand Down Expand Up @@ -389,9 +389,9 @@ func (a *ActivityLog) saveCurrentSegmentToStorageLocked(ctx context.Context, for

// Rotate to next segment
a.currentSegment.clientSequenceNumber += 1
if len(excessClients) > activitySegmentClientCapacity {
if len(excessClients) > ActivitySegmentClientCapacity {
a.logger.Warn("too many new active clients, dropping tail", "clients", len(excessClients))
excessClients = excessClients[:activitySegmentClientCapacity]
excessClients = excessClients[:ActivitySegmentClientCapacity]
}
a.currentSegment.currentClients.Clients = excessClients
err := a.saveCurrentSegmentInternal(ctx, force)
Expand Down
6 changes: 3 additions & 3 deletions vault/activity_log_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -784,8 +784,8 @@ func TestActivityLog_MultipleFragmentsAndSegments(t *testing.T) {
if err != nil {
t.Fatalf("could not unmarshal protobuf: %v", err)
}
if len(entityLog0.Clients) != activitySegmentClientCapacity {
t.Fatalf("unexpected client length. Expected %d, got %d", activitySegmentClientCapacity,
if len(entityLog0.Clients) != ActivitySegmentClientCapacity {
t.Fatalf("unexpected client length. Expected %d, got %d", ActivitySegmentClientCapacity,
len(entityLog0.Clients))
}

Expand All @@ -795,7 +795,7 @@ func TestActivityLog_MultipleFragmentsAndSegments(t *testing.T) {
if err != nil {
t.Fatalf("could not unmarshal protobuf: %v", err)
}
expectedCount := 8100 - activitySegmentClientCapacity
expectedCount := 8100 - ActivitySegmentClientCapacity
if len(entityLog1.Clients) != expectedCount {
t.Fatalf("unexpected entity length. Expected %d, got %d", expectedCount,
len(entityLog1.Clients))
Expand Down