feat: split large replay buffers before sending (#1305)

src/__tests__/extensions/replay/sessionrecording-utils.test.ts

@@ -5,6 +5,9 @@ import {
     CONSOLE_LOG_PLUGIN_NAME,
     PLUGIN_EVENT_TYPE,
     FULL_SNAPSHOT_EVENT_TYPE,
+    splitBuffer,
+    SEVEN_MEGABYTES,
+    estimateSize,
 } from '../../../extensions/replay/sessionrecording-utils'
 import { largeString, threeMBAudioURI, threeMBImageURI } from '../test_data/sessionrecording-utils-test-data'
 import { eventWithTime } from '@rrweb/types'
@@ -239,4 +242,95 @@ describe(`SessionRecording utility functions`, () => {
             })
         })
     })
+
+    describe('splitBuffer', () => {
+        it('should return the same buffer if size is less than SEVEN_MEGABYTES', () => {
+            const buffer = {
+                size: 5 * 1024 * 1024,
+                data: new Array(100).fill(0),
+                sessionId: 'session1',
+                windowId: 'window1',
+            }
+
+            const result = splitBuffer(buffer)
+            expect(result).toEqual([buffer])
+        })
+
+        it('should split the buffer into two halves if size is greater than or equal to SEVEN_MEGABYTES', () => {
+            const data = new Array(100).fill(0)
+            const expectedSize = estimateSize(new Array(50).fill(0))
+            const buffer = {
+                size: estimateSize(data),
+                data: data,
+                sessionId: 'session1',
+                windowId: 'window1',
+            }
+
+            // size limit just below the size of the buffer
+            const result = splitBuffer(buffer, 200)
+
+            expect(result).toHaveLength(2)
+            expect(result[0].data).toEqual(buffer.data.slice(0, 50))
+            expect(result[0].size).toEqual(expectedSize)
+            expect(result[1].data).toEqual(buffer.data.slice(50))
+            expect(result[1].size).toEqual(expectedSize)
+        })
+
+        it('should recursively split the buffer until each part is smaller than SEVEN_MEGABYTES', () => {
+            const largeDataArray = new Array(100).fill('a'.repeat(1024 * 1024))
+            const largeDataSize = estimateSize(largeDataArray) // >100mb
+            const buffer = {
+                size: largeDataSize,
+                data: largeDataArray,
+                sessionId: 'session1',
+                windowId: 'window1',
+            }
+
+            const result = splitBuffer(buffer)
+
+            expect(result.length).toBe(20)
+            let partTotal = 0
+            let sentArray: any[] = []
+            result.forEach((part) => {
+                expect(part.size).toBeLessThan(SEVEN_MEGABYTES)
+                sentArray = sentArray.concat(part.data)
+                partTotal += part.size
+            })
+
+            // it's a bit bigger because we have extra square brackets and commas when stringified
+            expect(partTotal).toBeGreaterThan(largeDataSize)
+            // but not much bigger!
+            expect(partTotal).toBeLessThan(largeDataSize * 1.001)
+            // we sent the same data overall
+            expect(JSON.stringify(sentArray)).toEqual(JSON.stringify(largeDataArray))
+        })
+
+        it('should handle buffer with size exactly SEVEN_MEGABYTES', () => {
+            const buffer = {
+                size: SEVEN_MEGABYTES,
+                data: new Array(100).fill(0),
+                sessionId: 'session1',
+                windowId: 'window1',
+            }
+
+            const result = splitBuffer(buffer)
+
+            expect(result).toHaveLength(2)
+            expect(result[0].data).toEqual(buffer.data.slice(0, 50))
+            expect(result[1].data).toEqual(buffer.data.slice(50))
+        })
+
+        it('should not split buffer if it has only one element', () => {
+            const buffer = {
+                size: 10 * 1024 * 1024,
+                data: [0],
+                sessionId: 'session1',
+                windowId: 'window1',
+            }
+
+            const result = splitBuffer(buffer)
+
+            expect(result).toEqual([buffer])
+        })
+    })
 })

src/extensions/replay/sessionrecording-utils.ts

@@ -12,6 +12,33 @@ import type {
 import type { DataURLOptions, MaskInputFn, MaskInputOptions, MaskTextFn, Mirror, SlimDOMOptions } from 'rrweb-snapshot'
 
 import { isObject } from '../../utils/type-utils'
+import { SnapshotBuffer } from './sessionrecording'
+
+// taken from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#circular_references
+function circularReferenceReplacer() {
+    const ancestors: any[] = []
+    return function (_key: string, value: any) {
+        if (isObject(value)) {
+            // `this` is the object that value is contained in,
+            // i.e., its direct parent.
+            // @ts-expect-error - TS was unhappy with `this` on the next line but the code is copied in from MDN
+            while (ancestors.length > 0 && ancestors.at(-1) !== this) {
+                ancestors.pop()
+            }
+            if (ancestors.includes(value)) {
+                return '[Circular]'
+            }
+            ancestors.push(value)
+            return value
+        } else {
+            return value
+        }
+    }
+}
+
+export function estimateSize(sizeable: unknown): number {
+    return JSON.stringify(sizeable, circularReferenceReplacer()).length
+}
 
 export const replacementImageURI =
     'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHJlY3Qgd2lkdGg9IjE2IiBoZWlnaHQ9IjE2IiBmaWxsPSJibGFjayIvPgo8cGF0aCBkPSJNOCAwSDE2TDAgMTZWOEw4IDBaIiBmaWxsPSIjMkQyRDJEIi8+CjxwYXRoIGQ9Ik0xNiA4VjE2SDhMMTYgOFoiIGZpbGw9IiMyRDJEMkQiLz4KPC9zdmc+Cg=='
@@ -135,3 +162,31 @@ export function truncateLargeConsoleLogs(_event: eventWithTime) {
     }
     return _event
 }
+
+export const SEVEN_MEGABYTES = 1024 * 1024 * 7 * 0.9 // ~7mb (with some wiggle room)
+
+// recursively splits large buffers into smaller ones
+// uses a pretty high size limit to avoid splitting too much
+export function splitBuffer(buffer: SnapshotBuffer, sizeLimit: number = SEVEN_MEGABYTES): SnapshotBuffer[] {
+    if (buffer.size >= sizeLimit && buffer.data.length > 1) {
+        const half = Math.floor(buffer.data.length / 2)
+        const firstHalf = buffer.data.slice(0, half)
+        const secondHalf = buffer.data.slice(half)
+        return [
+            splitBuffer({
+                size: estimateSize(firstHalf),
+                data: firstHalf,
+                sessionId: buffer.sessionId,
+                windowId: buffer.windowId,
+            }),
+            splitBuffer({
+                size: estimateSize(secondHalf),
+                data: secondHalf,
+                sessionId: buffer.sessionId,
+                windowId: buffer.windowId,
+            }),
+        ].flatMap((x) => x)
+    } else {
+        return [buffer]
+    }
+}

src/extensions/replay/sessionrecording.ts

@@ -8,11 +8,13 @@ import {
     SESSION_RECORDING_SAMPLE_RATE,
 } from '../../constants'
 import {
+    estimateSize,
     FULL_SNAPSHOT_EVENT_TYPE,
     INCREMENTAL_SNAPSHOT_EVENT_TYPE,
     META_EVENT_TYPE,
     recordOptions,
     rrwebRecord,
+    splitBuffer,
     truncateLargeConsoleLogs,
 } from './sessionrecording-utils'
 import { PostHog } from '../../posthog-core'
@@ -32,7 +34,7 @@ import {
     isUndefined,
 } from '../../utils/type-utils'
 import { logger } from '../../utils/logger'
-import { document, assignableWindow, window } from '../../utils/globals'
+import { assignableWindow, document, window } from '../../utils/globals'
 import { buildNetworkRequestOptions } from './config'
 import { isLocalhost } from '../../utils/request-utils'
 import { MutationRateLimiter } from './mutation-rate-limiter'
@@ -69,7 +71,7 @@ const ACTIVE_SOURCES = [
  */
 type SessionRecordingStatus = 'disabled' | 'sampled' | 'active' | 'buffering'
 
-interface SnapshotBuffer {
+export interface SnapshotBuffer {
     size: number
     data: any[]
     sessionId: string
@@ -91,32 +93,6 @@ const newQueuedEvent = (rrwebMethod: () => void): QueuedRRWebEvent => ({
 
 const LOGGER_PREFIX = '[SessionRecording]'
 
-// taken from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#circular_references
-function circularReferenceReplacer() {
-    const ancestors: any[] = []
-    return function (_key: string, value: any) {
-        if (isObject(value)) {
-            // `this` is the object that value is contained in,
-            // i.e., its direct parent.
-            // @ts-expect-error - TS was unhappy with `this` on the next line but the code is copied in from MDN
-            while (ancestors.length > 0 && ancestors.at(-1) !== this) {
-                ancestors.pop()
-            }
-            if (ancestors.includes(value)) {
-                return '[Circular]'
-            }
-            ancestors.push(value)
-            return value
-        } else {
-            return value
-        }
-    }
-}
-
-function estimateSize(event: eventWithTime): number {
-    return JSON.stringify(event, circularReferenceReplacer()).length
-}
-
 export class SessionRecording {
     private _endpoint: string
     private flushBufferTimer?: any
@@ -882,11 +858,14 @@ export class SessionRecording {
         }
 
         if (this.buffer.data.length > 0) {
-            this._captureSnapshot({
-                $snapshot_bytes: this.buffer.size,
-                $snapshot_data: this.buffer.data,
-                $session_id: this.buffer.sessionId,
-                $window_id: this.buffer.windowId,
+            const snapshotEvents = splitBuffer(this.buffer)
+            snapshotEvents.forEach((snapshotBuffer) => {
+                this._captureSnapshot({
+                    $snapshot_bytes: snapshotBuffer.size,
+                    $snapshot_data: snapshotBuffer.data,
+                    $session_id: snapshotBuffer.sessionId,
+                    $window_id: snapshotBuffer.windowId,
+                })
             })
         }