From 469043325c862b8e51bc6efcc8d1cc63ff6e89b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt?= Date: Fri, 21 Apr 2023 10:29:01 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20[RUMF-1530]=20enable=20sending=20re?= =?UTF-8?q?play=20metadata=20as=20json=20(#2177)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 6d439e56cc5f807e75096f41d65a2e2258cc3a8a. --- .../core/src/tools/experimentalFeatures.ts | 1 - packages/rum/src/boot/startRecording.spec.ts | 1 + .../buildReplayPayload.spec.ts | 77 +++---------------- .../segmentCollection/buildReplayPayload.ts | 35 +++------ packages/rum/test/readReplayPayload.ts | 22 +----- test/e2e/lib/framework/serverApps/intake.ts | 19 ++--- test/e2e/lib/types/serverEvents.ts | 5 +- .../scenario/recorder/recorder.scenario.ts | 71 +---------------- 8 files changed, 30 insertions(+), 201 deletions(-) diff --git a/packages/core/src/tools/experimentalFeatures.ts b/packages/core/src/tools/experimentalFeatures.ts index 76b12f532c..45cbb18250 100644 --- a/packages/core/src/tools/experimentalFeatures.ts +++ b/packages/core/src/tools/experimentalFeatures.ts @@ -16,7 +16,6 @@ export enum ExperimentalFeature { RESOURCE_PAGE_STATES = 'resource_page_states', COLLECT_FLUSH_REASON = 'collect_flush_reason', SANITIZE_INPUTS = 'sanitize_inputs', - REPLAY_JSON_PAYLOAD = 'replay_json_payload', } const enabledExperimentalFeatures: Set = new Set() diff --git a/packages/rum/src/boot/startRecording.spec.ts b/packages/rum/src/boot/startRecording.spec.ts index c92b55a301..ba0edfcf8d 100644 --- a/packages/rum/src/boot/startRecording.spec.ts +++ b/packages/rum/src/boot/startRecording.spec.ts @@ -91,6 +91,7 @@ describe('startRecording', () => { }, start: jasmine.any(Number), raw_segment_size: jasmine.any(Number), + compressed_segment_size: jasmine.any(Number), view: { id: 'view-id', }, diff --git a/packages/rum/src/domain/segmentCollection/buildReplayPayload.spec.ts b/packages/rum/src/domain/segmentCollection/buildReplayPayload.spec.ts index 1bbf57d06f..f5c630a700 100644 --- a/packages/rum/src/domain/segmentCollection/buildReplayPayload.spec.ts +++ b/packages/rum/src/domain/segmentCollection/buildReplayPayload.spec.ts @@ -1,8 +1,8 @@ import pako from 'pako' -import { addExperimentalFeatures, ExperimentalFeature, isIE, resetExperimentalFeatures } from '@datadog/browser-core' +import { isIE } from '@datadog/browser-core' import type { BrowserSegment, BrowserSegmentMetadata } from '../../types' import { readReplayPayload } from '../../../test' -import { buildReplayPayload, toFormEntries } from './buildReplayPayload' +import { buildReplayPayload } from './buildReplayPayload' describe('buildReplayPayload', () => { const SEGMENT = { foo: 'bar' } as unknown as BrowserSegment @@ -40,40 +40,14 @@ describe('buildReplayPayload', () => { expect(segment).toEqual(SEGMENT) }) - describe('with replay_json_payload experimental flag', () => { - beforeEach(() => { - addExperimentalFeatures([ExperimentalFeature.REPLAY_JSON_PAYLOAD]) - }) - - afterEach(() => { - resetExperimentalFeatures() - }) - - it('adds the metadata plus the segment sizes as the `event` entry', async () => { - const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length) - const eventEntry = (payload.data as FormData).get('event')! as File - expect(eventEntry.size).toBe(JSON.stringify(METADATA_AND_SEGMENT_SIZES).length) - expect(eventEntry.name).toBe('blob') - expect(eventEntry.type).toBe('application/json') - const { metadata } = await readReplayPayload(payload) - expect(metadata).toEqual(METADATA_AND_SEGMENT_SIZES) - }) - }) - - describe('without replay_json_payload experimental flag', () => { - it('adds the metadata plus the segment sizes as the `event` entry', () => { - const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length) - const formData = payload.data as FormData - expect(formData.get('application.id')).toBe(METADATA.application.id) - expect(formData.get('session.id')).toBe(METADATA.session.id) - expect(formData.get('view.id')).toBe(METADATA.view.id) - expect(formData.get('start')).toBe(String(METADATA.start)) - expect(formData.get('end')).toBe(String(METADATA.end)) - expect(formData.get('records_count')).toBe(String(METADATA.records_count)) - expect(formData.get('source')).toBe(METADATA.source) - expect(formData.get('creation_reason')).toBe(METADATA.creation_reason) - expect(formData.get('raw_segment_size')).toBe(String(SERIALIZED_SEGMENT.length)) - }) + it('adds the metadata plus the segment sizes as the `event` entry', async () => { + const payload = buildReplayPayload(COMPRESSED_SEGMENT, METADATA, SERIALIZED_SEGMENT.length) + const eventEntry = (payload.data as FormData).get('event')! as File + expect(eventEntry.size).toBe(JSON.stringify(METADATA_AND_SEGMENT_SIZES).length) + expect(eventEntry.name).toBe('blob') + expect(eventEntry.type).toBe('application/json') + const { metadata } = await readReplayPayload(payload) + expect(metadata).toEqual(METADATA_AND_SEGMENT_SIZES) }) it('returns the approximate byte counts of the request', () => { @@ -81,34 +55,3 @@ describe('buildReplayPayload', () => { expect(payload.bytesCount).toBe(COMPRESSED_SEGMENT.byteLength) }) }) - -describe('toFormEntries', () => { - let callbackSpy: jasmine.Spy<(key: string, value: string) => void> - beforeEach(() => { - callbackSpy = jasmine.createSpy() - }) - - it('handles top level properties', () => { - toFormEntries({ foo: 'bar', zig: 'zag' }, callbackSpy) - expect(callbackSpy.calls.allArgs()).toEqual([ - ['foo', 'bar'], - ['zig', 'zag'], - ]) - }) - - it('handles nested properties', () => { - toFormEntries({ foo: { bar: 'baz', zig: { zag: 'zug' } } }, callbackSpy) - expect(callbackSpy.calls.allArgs()).toEqual([ - ['foo.bar', 'baz'], - ['foo.zig.zag', 'zug'], - ]) - }) - - it('converts values to string', () => { - toFormEntries({ foo: 42, bar: null }, callbackSpy) - expect(callbackSpy.calls.allArgs()).toEqual([ - ['foo', '42'], - ['bar', 'null'], - ]) - }) -}) diff --git a/packages/rum/src/domain/segmentCollection/buildReplayPayload.ts b/packages/rum/src/domain/segmentCollection/buildReplayPayload.ts index daa0328bcf..0fa759fe5c 100644 --- a/packages/rum/src/domain/segmentCollection/buildReplayPayload.ts +++ b/packages/rum/src/domain/segmentCollection/buildReplayPayload.ts @@ -1,5 +1,5 @@ import type { Payload } from '@datadog/browser-core' -import { ExperimentalFeature, isExperimentalFeatureEnabled, objectEntries, assign } from '@datadog/browser-core' +import { assign } from '@datadog/browser-core' import type { BrowserSegmentMetadata } from '../../types' export type BrowserSegmentMetadataAndSegmentSizes = BrowserSegmentMetadata & { @@ -22,30 +22,15 @@ export function buildReplayPayload( `${metadata.session.id}-${metadata.start}` ) - if (isExperimentalFeatureEnabled(ExperimentalFeature.REPLAY_JSON_PAYLOAD)) { - const metadataAndSegmentSizes: BrowserSegmentMetadataAndSegmentSizes = assign( - { - raw_segment_size: rawSegmentBytesCount, - compressed_segment_size: data.byteLength, - }, - metadata - ) - const serializedMetadataAndSegmentSizes = JSON.stringify(metadataAndSegmentSizes) - formData.append('event', new Blob([serializedMetadataAndSegmentSizes], { type: 'application/json' })) - } else { - toFormEntries(metadata, (key, value) => formData.append(key, value)) - formData.append('raw_segment_size', rawSegmentBytesCount.toString()) - } + const metadataAndSegmentSizes: BrowserSegmentMetadataAndSegmentSizes = assign( + { + raw_segment_size: rawSegmentBytesCount, + compressed_segment_size: data.byteLength, + }, + metadata + ) + const serializedMetadataAndSegmentSizes = JSON.stringify(metadataAndSegmentSizes) + formData.append('event', new Blob([serializedMetadataAndSegmentSizes], { type: 'application/json' })) return { data: formData, bytesCount: data.byteLength } } - -export function toFormEntries(input: object, onEntry: (key: string, value: string) => void, prefix = '') { - objectEntries(input as { [key: string]: unknown }).forEach(([key, value]) => { - if (typeof value === 'object' && value !== null) { - toFormEntries(value, onEntry, `${prefix}${key}.`) - } else { - onEntry(`${prefix}${key}`, String(value)) - } - }) -} diff --git a/packages/rum/test/readReplayPayload.ts b/packages/rum/test/readReplayPayload.ts index 40ea969ffc..a60bc2234a 100644 --- a/packages/rum/test/readReplayPayload.ts +++ b/packages/rum/test/readReplayPayload.ts @@ -1,7 +1,7 @@ import pako from 'pako' import type { Payload } from '@datadog/browser-core' -import type { BrowserSegment, CreationReason } from '../src/types' +import type { BrowserSegment } from '../src/types' import type { BrowserSegmentMetadataAndSegmentSizes } from '../src/domain/segmentCollection' export async function readReplayPayload(payload: Payload) { @@ -18,26 +18,6 @@ function readSegmentFromReplayPayload(payload: Payload) { } export function readMetadataFromReplayPayload(payload: Payload) { - const formData = payload.data as FormData - if (!formData.has('event')) { - // TODO remove this when replay_json_payload is enabled - return { - application: { id: formData.get('application.id') as string }, - session: { id: formData.get('session.id') as string }, - view: { id: formData.get('view.id') as string }, - start: Number(formData.get('start')), - end: Number(formData.get('end')), - records_count: Number(formData.get('records_count')), - source: formData.get('source') as 'browser', - creation_reason: formData.get('creation_reason') as CreationReason, - raw_segment_size: Number(formData.get('raw_segment_size')), - index_in_view: Number(formData.get('index_in_view')), - has_full_snapshot: formData.get('has_full_snapshot') === 'true', - } satisfies Omit & { - compressed_segment_size?: number - } - } - return readJsonBlob((payload.data as FormData).get('event') as Blob) as Promise } diff --git a/test/e2e/lib/framework/serverApps/intake.ts b/test/e2e/lib/framework/serverApps/intake.ts index d16e967148..d48722f5c0 100644 --- a/test/e2e/lib/framework/serverApps/intake.ts +++ b/test/e2e/lib/framework/serverApps/intake.ts @@ -95,12 +95,7 @@ function forwardEventsToIntake(req: express.Request): Promise { function storeReplayData(req: express.Request, events: EventRegistry): Promise { return new Promise((resolve, reject) => { let segmentPromise: Promise - let metadataFromJsonPayloadPromise: Promise - - // TODO: remove this when enabling replay_json_payload - const metadataFromMultipartFields: { - [field: string]: string - } = {} + let metadataPromise: Promise req.busboy.on('file', (name, stream, info) => { const { filename, encoding, mimeType } = info @@ -112,20 +107,16 @@ function storeReplayData(req: express.Request, events: EventRegistry): Promise JSON.parse(data.toString()) as BrowserSegmentMetadataAndSegmentSizes ) } }) - req.busboy.on('field', (key: string, value: string) => { - metadataFromMultipartFields[key] = value - }) - req.busboy.on('finish', () => { - Promise.all([segmentPromise, metadataFromJsonPayloadPromise]) - .then(([segment, metadataFromJsonPayload]) => { - events.push('sessionReplay', { metadata: metadataFromJsonPayload || metadataFromMultipartFields, segment }) + Promise.all([segmentPromise, metadataPromise]) + .then(([segment, metadata]) => { + events.push('sessionReplay', { metadata, segment }) }) .then(resolve) .catch((e) => reject(e)) diff --git a/test/e2e/lib/types/serverEvents.ts b/test/e2e/lib/types/serverEvents.ts index 75ee26a288..f6a3cddd34 100644 --- a/test/e2e/lib/types/serverEvents.ts +++ b/test/e2e/lib/types/serverEvents.ts @@ -36,8 +36,5 @@ export interface SegmentFile { export interface SessionReplayCall { segment: SegmentFile - metadata: - | BrowserSegmentMetadataAndSegmentSizes - // TODO: remove this when enabling replay_json_payload - | Record + metadata: BrowserSegmentMetadataAndSegmentSizes } diff --git a/test/e2e/scenario/recorder/recorder.scenario.ts b/test/e2e/scenario/recorder/recorder.scenario.ts index bd2987020a..bead027797 100644 --- a/test/e2e/scenario/recorder/recorder.scenario.ts +++ b/test/e2e/scenario/recorder/recorder.scenario.ts @@ -1,10 +1,4 @@ -import type { - InputData, - StyleSheetRuleData, - BrowserSegment, - ScrollData, - CreationReason, -} from '@datadog/browser-rum/src/types' +import type { InputData, StyleSheetRuleData, BrowserSegment, ScrollData } from '@datadog/browser-rum/src/types' import { NodeType, IncrementalSource, MouseInteractionType } from '@datadog/browser-rum/src/types' // Import from src to have properties of const enums @@ -24,14 +18,10 @@ import { findMouseInteractionRecords, findElementWithTagName, } from '@datadog/browser-rum/test' -import { ExperimentalFeature } from '@datadog/browser-core/src/tools/experimentalFeatures' -import type { BrowserSegmentMetadataAndSegmentSizes } from '@datadog/browser-rum/src/domain/segmentCollection' import { flushEvents, createTest, bundleSetup, html } from '../../lib/framework' import { browserExecute, browserExecuteAsync } from '../../lib/helpers/browser' import { getFirstSegment, getLastSegment, initRumAndStartRecording } from '../../lib/helpers/replay' -import type { SegmentFile } from '../../lib/types/serverEvents' -const INTEGER_RE = /^\d+$/ const TIMESTAMP_RE = /^\d{13}$/ const UUID_RE = /^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/ @@ -46,64 +36,7 @@ describe('recorder', () => { await flushEvents() expect(serverEvents.sessionReplay.length).toBe(1) - const { segment, metadata } = serverEvents.sessionReplay[0] as { - segment: SegmentFile - metadata: Record - } - expect(metadata).toEqual({ - 'application.id': jasmine.stringMatching(UUID_RE), - creation_reason: 'init', - end: jasmine.stringMatching(TIMESTAMP_RE), - has_full_snapshot: 'true', - records_count: jasmine.stringMatching(INTEGER_RE), - 'session.id': jasmine.stringMatching(UUID_RE), - start: jasmine.stringMatching(TIMESTAMP_RE), - 'view.id': jasmine.stringMatching(UUID_RE), - raw_segment_size: jasmine.stringMatching(INTEGER_RE), - index_in_view: '0', - source: 'browser', - }) - expect(segment).toEqual({ - data: { - application: { id: metadata['application.id'] }, - creation_reason: metadata.creation_reason as CreationReason, - end: Number(metadata.end), - has_full_snapshot: true, - records: jasmine.any(Array), - records_count: Number(metadata.records_count), - session: { id: metadata['session.id'] }, - start: Number(metadata.start), - view: { id: metadata['view.id'] }, - index_in_view: 0, - source: 'browser', - }, - encoding: jasmine.any(String), - filename: `${metadata['session.id']}-${metadata.start}`, - mimetype: 'application/octet-stream', - }) - expect(findMeta(segment.data)).toBeTruthy('have a Meta record') - expect(findFullSnapshot(segment.data)).toBeTruthy('have a FullSnapshot record') - expect(findIncrementalSnapshot(segment.data, IncrementalSource.MouseInteraction)).toBeTruthy( - 'have a IncrementalSnapshot/MouseInteraction record' - ) - }) - - createTest('record mouse move with replay_json_payload experimental feature') - .withRum({ - enableExperimentalFeatures: [ExperimentalFeature.REPLAY_JSON_PAYLOAD], - }) - .withRumInit(initRumAndStartRecording) - .run(async ({ serverEvents }) => { - await browserExecute(() => document.documentElement.outerHTML) - const html = await $('html') - await html.click() - await flushEvents() - - expect(serverEvents.sessionReplay.length).toBe(1) - const { segment, metadata } = serverEvents.sessionReplay[0] as { - segment: SegmentFile - metadata: BrowserSegmentMetadataAndSegmentSizes - } + const { segment, metadata } = serverEvents.sessionReplay[0] expect(metadata).toEqual({ application: { id: jasmine.stringMatching(UUID_RE) }, creation_reason: 'init',