✨ [RUMF-1530] enable sending replay metadata as json (#2177)

This reverts commit 6d439e5.
DataDog · Apr 21, 2023 · 4690433 · 4690433
1 parent fb1c78e
commit 4690433
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 201 deletions.
diff --git a/packages/core/src/tools/experimentalFeatures.ts b/packages/core/src/tools/experimentalFeatures.ts
@@ -16,7 +16,6 @@ export enum ExperimentalFeature {
   RESOURCE_PAGE_STATES = 'resource_page_states',
   COLLECT_FLUSH_REASON = 'collect_flush_reason',
   SANITIZE_INPUTS = 'sanitize_inputs',
-  REPLAY_JSON_PAYLOAD = 'replay_json_payload',
 }
 
 const enabledExperimentalFeatures: Set<ExperimentalFeature> = new Set()

diff --git a/packages/rum/src/boot/startRecording.spec.ts b/packages/rum/src/boot/startRecording.spec.ts
@@ -91,6 +91,7 @@ describe('startRecording', () => {
       },
       start: jasmine.any(Number),
       raw_segment_size: jasmine.any(Number),
+      compressed_segment_size: jasmine.any(Number),
       view: {
         id: 'view-id',
       },

diff --git a/packages/rum/src/domain/segmentCollection/buildReplayPayload.spec.ts b/packages/rum/src/domain/segmentCollection/buildReplayPayload.spec.ts
@@ -1,8 +1,8 @@
 import pako from 'pako'
-import { addExperimentalFeatures, ExperimentalFeature, isIE, resetExperimentalFeatures } from '@datadog/browser-core'
+import { isIE } from '@datadog/browser-core'
 import type { BrowserSegment, BrowserSegmentMetadata } from '../../types'
 import { readReplayPayload } from '../../../test'
-import { buildReplayPayload, toFormEntries } from './buildReplayPayload'
+import { buildReplayPayload } from './buildReplayPayload'
 
 describe('buildReplayPayload', () => {
   const SEGMENT = { foo: 'bar' } as unknown as BrowserSegment
@@ -40,75 +40,18 @@ describe('buildReplayPayload', () => {
     expect(segment).toEqual(SEGMENT)
   })
 
-  describe('with replay_json_payload experimental flag', () => {
-    beforeEach(() => {
-      addExperimentalFeatures([ExperimentalFeature.REPLAY_JSON_PAYLOAD])
-    })
-
-    afterEach(() => {
-      resetExperimentalFeatures()
-    })
-
-    it('adds the metadata plus the segment sizes as the `event` entry', async () => {
-      const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length)
-      const eventEntry = (payload.data as FormData).get('event')! as File
-      expect(eventEntry.size).toBe(JSON.stringify(METADATA_AND_SEGMENT_SIZES).length)
-      expect(eventEntry.name).toBe('blob')
-      expect(eventEntry.type).toBe('application/json')
-      const { metadata } = await readReplayPayload(payload)
-      expect(metadata).toEqual(METADATA_AND_SEGMENT_SIZES)
-    })
-  })
-
-  describe('without replay_json_payload experimental flag', () => {
-    it('adds the metadata plus the segment sizes as the `event` entry', () => {
-      const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length)
-      const formData = payload.data as FormData
-      expect(formData.get('application.id')).toBe(METADATA.application.id)
-      expect(formData.get('session.id')).toBe(METADATA.session.id)
-      expect(formData.get('view.id')).toBe(METADATA.view.id)
-      expect(formData.get('start')).toBe(String(METADATA.start))
-      expect(formData.get('end')).toBe(String(METADATA.end))
-      expect(formData.get('records_count')).toBe(String(METADATA.records_count))
-      expect(formData.get('source')).toBe(METADATA.source)
-      expect(formData.get('creation_reason')).toBe(METADATA.creation_reason)
-      expect(formData.get('raw_segment_size')).toBe(String(SERIALIZED_SEGMENT.length))
-    })
+  it('adds the metadata plus the segment sizes as the `event` entry', async () => {
+    const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length)
+    const eventEntry = (payload.data as FormData).get('event')! as File
+    expect(eventEntry.size).toBe(JSON.stringify(METADATA_AND_SEGMENT_SIZES).length)
+    expect(eventEntry.name).toBe('blob')
+    expect(eventEntry.type).toBe('application/json')
+    const { metadata } = await readReplayPayload(payload)
+    expect(metadata).toEqual(METADATA_AND_SEGMENT_SIZES)
   })
 
   it('returns the approximate byte counts of the request', () => {
     const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length)
     expect(payload.bytesCount).toBe(COMPRESSED_SEGMENT.byteLength)
   })
 })
-
-describe('toFormEntries', () => {
-  let callbackSpy: jasmine.Spy<(key: string, value: string) => void>
-  beforeEach(() => {
-    callbackSpy = jasmine.createSpy()
-  })
-
-  it('handles top level properties', () => {
-    toFormEntries({ foo: 'bar', zig: 'zag' }, callbackSpy)
-    expect(callbackSpy.calls.allArgs()).toEqual([
-      ['foo', 'bar'],
-      ['zig', 'zag'],
-    ])
-  })
-
-  it('handles nested properties', () => {
-    toFormEntries({ foo: { bar: 'baz', zig: { zag: 'zug' } } }, callbackSpy)
-    expect(callbackSpy.calls.allArgs()).toEqual([
-      ['foo.bar', 'baz'],
-      ['foo.zig.zag', 'zug'],
-    ])
-  })
-
-  it('converts values to string', () => {
-    toFormEntries({ foo: 42, bar: null }, callbackSpy)
-    expect(callbackSpy.calls.allArgs()).toEqual([
-      ['foo', '42'],
-      ['bar', 'null'],
-    ])
-  })
-})
diff --git a/packages/rum/src/domain/segmentCollection/buildReplayPayload.ts b/packages/rum/src/domain/segmentCollection/buildReplayPayload.ts
@@ -1,5 +1,5 @@
 import type { Payload } from '@datadog/browser-core'
-import { ExperimentalFeature, isExperimentalFeatureEnabled, objectEntries, assign } from '@datadog/browser-core'
+import { assign } from '@datadog/browser-core'
 import type { BrowserSegmentMetadata } from '../../types'
 
 export type BrowserSegmentMetadataAndSegmentSizes = BrowserSegmentMetadata & {
@@ -22,30 +22,15 @@ export function buildReplayPayload(
     `${metadata.session.id}-${metadata.start}`
   )
 
-  if (isExperimentalFeatureEnabled(ExperimentalFeature.REPLAY_JSON_PAYLOAD)) {
-    const metadataAndSegmentSizes: BrowserSegmentMetadataAndSegmentSizes = assign(
-      {
-        raw_segment_size: rawSegmentBytesCount,
-        compressed_segment_size: data.byteLength,
-      },
-      metadata
-    )
-    const serializedMetadataAndSegmentSizes = JSON.stringify(metadataAndSegmentSizes)
-    formData.append('event', new Blob([serializedMetadataAndSegmentSizes], { type: 'application/json' }))
-  } else {
-    toFormEntries(metadata, (key, value) => formData.append(key, value))
-    formData.append('raw_segment_size', rawSegmentBytesCount.toString())
-  }
+  const metadataAndSegmentSizes: BrowserSegmentMetadataAndSegmentSizes = assign(
+    {
+      raw_segment_size: rawSegmentBytesCount,
+      compressed_segment_size: data.byteLength,
+    },
+    metadata
+  )
+  const serializedMetadataAndSegmentSizes = JSON.stringify(metadataAndSegmentSizes)
+  formData.append('event', new Blob([serializedMetadataAndSegmentSizes], { type: 'application/json' }))
 
   return { data: formData, bytesCount: data.byteLength }
 }
-
-export function toFormEntries(input: object, onEntry: (key: string, value: string) => void, prefix = '') {
-  objectEntries(input as { [key: string]: unknown }).forEach(([key, value]) => {
-    if (typeof value === 'object' && value !== null) {
-      toFormEntries(value, onEntry, `${prefix}${key}.`)
-    } else {
-      onEntry(`${prefix}${key}`, String(value))
-    }
-  })
-}
diff --git a/packages/rum/test/readReplayPayload.ts b/packages/rum/test/readReplayPayload.ts
@@ -1,7 +1,7 @@
 import pako from 'pako'
 
 import type { Payload } from '@datadog/browser-core'
-import type { BrowserSegment, CreationReason } from '../src/types'
+import type { BrowserSegment } from '../src/types'
 import type { BrowserSegmentMetadataAndSegmentSizes } from '../src/domain/segmentCollection'
 
 export async function readReplayPayload(payload: Payload) {
@@ -18,26 +18,6 @@ function readSegmentFromReplayPayload(payload: Payload) {
 }
 
 export function readMetadataFromReplayPayload(payload: Payload) {
-  const formData = payload.data as FormData
-  if (!formData.has('event')) {
-    // TODO remove this when replay_json_payload is enabled
-    return {
-      application: { id: formData.get('application.id') as string },
-      session: { id: formData.get('session.id') as string },
-      view: { id: formData.get('view.id') as string },
-      start: Number(formData.get('start')),
-      end: Number(formData.get('end')),
-      records_count: Number(formData.get('records_count')),
-      source: formData.get('source') as 'browser',
-      creation_reason: formData.get('creation_reason') as CreationReason,
-      raw_segment_size: Number(formData.get('raw_segment_size')),
-      index_in_view: Number(formData.get('index_in_view')),
-      has_full_snapshot: formData.get('has_full_snapshot') === 'true',
-    } satisfies Omit<BrowserSegmentMetadataAndSegmentSizes, 'compressed_segment_size'> & {
-      compressed_segment_size?: number
-    }
-  }
-
   return readJsonBlob((payload.data as FormData).get('event') as Blob) as Promise<BrowserSegmentMetadataAndSegmentSizes>
 }
 

diff --git a/test/e2e/lib/framework/serverApps/intake.ts b/test/e2e/lib/framework/serverApps/intake.ts
@@ -95,12 +95,7 @@ function forwardEventsToIntake(req: express.Request): Promise<any> {
 function storeReplayData(req: express.Request, events: EventRegistry): Promise<any> {
   return new Promise((resolve, reject) => {
     let segmentPromise: Promise<SegmentFile>
-    let metadataFromJsonPayloadPromise: Promise<BrowserSegmentMetadataAndSegmentSizes>
-
-    // TODO: remove this when enabling replay_json_payload
-    const metadataFromMultipartFields: {
-      [field: string]: string
-    } = {}
+    let metadataPromise: Promise<BrowserSegmentMetadataAndSegmentSizes>
 
     req.busboy.on('file', (name, stream, info) => {
       const { filename, encoding, mimeType } = info
@@ -112,20 +107,16 @@ function storeReplayData(req: express.Request, events: EventRegistry): Promise<a
           data: JSON.parse(data.toString()),
         }))
       } else if (name === 'event') {
-        metadataFromJsonPayloadPromise = readStream(stream).then(
+        metadataPromise = readStream(stream).then(
           (data) => JSON.parse(data.toString()) as BrowserSegmentMetadataAndSegmentSizes
         )
       }
     })
 
-    req.busboy.on('field', (key: string, value: string) => {
-      metadataFromMultipartFields[key] = value
-    })
-
     req.busboy.on('finish', () => {
-      Promise.all([segmentPromise, metadataFromJsonPayloadPromise])
-        .then(([segment, metadataFromJsonPayload]) => {
-          events.push('sessionReplay', { metadata: metadataFromJsonPayload || metadataFromMultipartFields, segment })
+      Promise.all([segmentPromise, metadataPromise])
+        .then(([segment, metadata]) => {
+          events.push('sessionReplay', { metadata, segment })
         })
         .then(resolve)
         .catch((e) => reject(e))

diff --git a/test/e2e/lib/types/serverEvents.ts b/test/e2e/lib/types/serverEvents.ts
@@ -36,8 +36,5 @@ export interface SegmentFile {
 
 export interface SessionReplayCall {
   segment: SegmentFile
-  metadata:
-    | BrowserSegmentMetadataAndSegmentSizes
-    // TODO: remove this when enabling replay_json_payload
-    | Record<string, string>
+  metadata: BrowserSegmentMetadataAndSegmentSizes
 }
diff --git a/test/e2e/scenario/recorder/recorder.scenario.ts b/test/e2e/scenario/recorder/recorder.scenario.ts
@@ -1,10 +1,4 @@
-import type {
-  InputData,
-  StyleSheetRuleData,
-  BrowserSegment,
-  ScrollData,
-  CreationReason,
-} from '@datadog/browser-rum/src/types'
+import type { InputData, StyleSheetRuleData, BrowserSegment, ScrollData } from '@datadog/browser-rum/src/types'
 import { NodeType, IncrementalSource, MouseInteractionType } from '@datadog/browser-rum/src/types'
 
 // Import from src to have properties of const enums
@@ -24,14 +18,10 @@ import {
   findMouseInteractionRecords,
   findElementWithTagName,
 } from '@datadog/browser-rum/test'
-import { ExperimentalFeature } from '@datadog/browser-core/src/tools/experimentalFeatures'
-import type { BrowserSegmentMetadataAndSegmentSizes } from '@datadog/browser-rum/src/domain/segmentCollection'
 import { flushEvents, createTest, bundleSetup, html } from '../../lib/framework'
 import { browserExecute, browserExecuteAsync } from '../../lib/helpers/browser'
 import { getFirstSegment, getLastSegment, initRumAndStartRecording } from '../../lib/helpers/replay'
-import type { SegmentFile } from '../../lib/types/serverEvents'
 
-const INTEGER_RE = /^\d+$/
 const TIMESTAMP_RE = /^\d{13}$/
 const UUID_RE = /^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/
 
@@ -46,64 +36,7 @@ describe('recorder', () => {
       await flushEvents()
 
       expect(serverEvents.sessionReplay.length).toBe(1)
-      const { segment, metadata } = serverEvents.sessionReplay[0] as {
-        segment: SegmentFile
-        metadata: Record<string, string>
-      }
-      expect(metadata).toEqual({
-        'application.id': jasmine.stringMatching(UUID_RE),
-        creation_reason: 'init',
-        end: jasmine.stringMatching(TIMESTAMP_RE),
-        has_full_snapshot: 'true',
-        records_count: jasmine.stringMatching(INTEGER_RE),
-        'session.id': jasmine.stringMatching(UUID_RE),
-        start: jasmine.stringMatching(TIMESTAMP_RE),
-        'view.id': jasmine.stringMatching(UUID_RE),
-        raw_segment_size: jasmine.stringMatching(INTEGER_RE),
-        index_in_view: '0',
-        source: 'browser',
-      })
-      expect(segment).toEqual({
-        data: {
-          application: { id: metadata['application.id'] },
-          creation_reason: metadata.creation_reason as CreationReason,
-          end: Number(metadata.end),
-          has_full_snapshot: true,
-          records: jasmine.any(Array),
-          records_count: Number(metadata.records_count),
-          session: { id: metadata['session.id'] },
-          start: Number(metadata.start),
-          view: { id: metadata['view.id'] },
-          index_in_view: 0,
-          source: 'browser',
-        },
-        encoding: jasmine.any(String),
-        filename: `${metadata['session.id']}-${metadata.start}`,
-        mimetype: 'application/octet-stream',
-      })
-      expect(findMeta(segment.data)).toBeTruthy('have a Meta record')
-      expect(findFullSnapshot(segment.data)).toBeTruthy('have a FullSnapshot record')
-      expect(findIncrementalSnapshot(segment.data, IncrementalSource.MouseInteraction)).toBeTruthy(
-        'have a IncrementalSnapshot/MouseInteraction record'
-      )
-    })
-
-  createTest('record mouse move with replay_json_payload experimental feature')
-    .withRum({
-      enableExperimentalFeatures: [ExperimentalFeature.REPLAY_JSON_PAYLOAD],
-    })
-    .withRumInit(initRumAndStartRecording)
-    .run(async ({ serverEvents }) => {
-      await browserExecute(() => document.documentElement.outerHTML)
-      const html = await $('html')
-      await html.click()
-      await flushEvents()
-
-      expect(serverEvents.sessionReplay.length).toBe(1)
-      const { segment, metadata } = serverEvents.sessionReplay[0] as {
-        segment: SegmentFile
-        metadata: BrowserSegmentMetadataAndSegmentSizes
-      }
+      const { segment, metadata } = serverEvents.sessionReplay[0]
       expect(metadata).toEqual({
         application: { id: jasmine.stringMatching(UUID_RE) },
         creation_reason: 'init',