Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✅ [RUM-253] add E2E tests related to compression #2416

Merged
merged 3 commits into from
Oct 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 13 additions & 9 deletions test/e2e/lib/framework/intakeRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,31 @@ import type { TelemetryEvent, TelemetryErrorEvent, TelemetryConfigurationEvent }
import type { BrowserSegment } from '@datadog/browser-rum/src/types'
import type { BrowserSegmentMetadataAndSegmentSizes } from '@datadog/browser-rum/src/domain/segmentCollection'

type BaseIntakeRequest = {
isBridge: boolean
encoding: string | null
}

export type LogsIntakeRequest = {
intakeType: 'logs'
isBridge: boolean
events: LogsEvent[]
}
} & BaseIntakeRequest

export type RumIntakeRequest = {
intakeType: 'rum'
isBridge: boolean
events: Array<RumEvent | TelemetryEvent>
}
} & BaseIntakeRequest

export type ReplayIntakeRequest = {
intakeType: 'replay'
isBridge: false
segment: BrowserSegment
metadata: BrowserSegmentMetadataAndSegmentSizes
filename: string
encoding: string
mimetype: string
}
segmentFile: {
filename: string
encoding: string
mimetype: string
}
} & BaseIntakeRequest

export type IntakeRequest = LogsIntakeRequest | RumIntakeRequest | ReplayIntakeRequest

Expand Down
54 changes: 38 additions & 16 deletions test/e2e/lib/framework/serverApps/intake.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
import { createInflate } from 'zlib'
import { createInflate, inflateSync } from 'zlib'
import https from 'https'
import connectBusboy from 'connect-busboy'
import express from 'express'

import cors from 'cors'
import type { BrowserSegmentMetadataAndSegmentSizes } from '@datadog/browser-rum/src/domain/segmentCollection'
import type { BrowserSegment } from '@datadog/browser-rum/src/types'
import type { IntakeRegistry, IntakeRequest, ReplayIntakeRequest } from '../intakeRegistry'
import type {
IntakeRegistry,
IntakeRequest,
LogsIntakeRequest,
ReplayIntakeRequest,
RumIntakeRequest,
} from '../intakeRegistry'

interface IntakeRequestInfos {
isBridge: boolean
intakeType: IntakeRequest['intakeType']
encoding: string | null
}

export function createIntakeServerApp(intakeRegistry: IntakeRegistry) {
Expand Down Expand Up @@ -42,45 +49,60 @@ function computeIntakeRequestInfos(req: express.Request): IntakeRequestInfos {
if (!ddforward) {
throw new Error('ddforward is missing')
}
const { pathname, searchParams } = new URL(ddforward, 'https://example.org')

const encoding = req.headers['content-encoding'] || searchParams.get('dd-evp-encoding')

if (req.query.bridge === 'true') {
const eventType = req.query.event_type
return {
isBridge: true,
encoding,
intakeType: eventType === 'log' ? 'logs' : 'rum',
}
}

let intakeType: IntakeRequest['intakeType']
// ddforward = /api/v2/rum?key=value
const endpoint = ddforward.split(/[/?]/)[3]
// pathname = /api/v2/rum
const endpoint = pathname.split(/[/?]/)[3]
if (endpoint === 'logs' || endpoint === 'rum' || endpoint === 'replay') {
intakeType = endpoint
} else {
throw new Error("Can't find intake type")
}
return {
isBridge: false,
encoding,
intakeType,
}
}

async function readIntakeRequest(req: express.Request, infos: IntakeRequestInfos): Promise<IntakeRequest> {
if (infos.intakeType === 'replay') {
return readReplayIntakeRequest(req)
}
function readIntakeRequest(req: express.Request, infos: IntakeRequestInfos): Promise<IntakeRequest> {
return infos.intakeType === 'replay'
? readReplayIntakeRequest(req, infos as IntakeRequestInfos & { intakeType: 'replay' })
: readRumOrLogsIntakeRequest(req, infos as IntakeRequestInfos & { intakeType: 'rum' | 'logs' })
}

async function readRumOrLogsIntakeRequest(
req: express.Request,
infos: IntakeRequestInfos & { intakeType: 'rum' | 'logs' }
): Promise<RumIntakeRequest | LogsIntakeRequest> {
const rawBody = await readStream(req)
const encodedBody = infos.encoding === 'deflate' ? inflateSync(rawBody) : rawBody

return {
intakeType: infos.intakeType,
isBridge: infos.isBridge,
events: (await readStream(req))
...infos,
events: encodedBody
.toString('utf-8')
.split('\n')
.map((line): any => JSON.parse(line)),
}
}

function readReplayIntakeRequest(req: express.Request): Promise<ReplayIntakeRequest> {
function readReplayIntakeRequest(
req: express.Request,
infos: IntakeRequestInfos & { intakeType: 'replay' }
): Promise<ReplayIntakeRequest> {
return new Promise((resolve, reject) => {
let segmentPromise: Promise<{
encoding: string
Expand Down Expand Up @@ -108,11 +130,11 @@ function readReplayIntakeRequest(req: express.Request): Promise<ReplayIntakeRequ

req.busboy.on('finish', () => {
Promise.all([segmentPromise, metadataPromise])
.then(([segmentEntry, metadata]) => ({
intakeType: 'replay' as const,
isBridge: false as const,
.then(([{ segment, ...segmentFile }, metadata]) => ({
...infos,
segmentFile,
metadata,
...segmentEntry,
segment,
}))
.then(resolve, reject)
})
Expand Down
6 changes: 5 additions & 1 deletion test/e2e/scenario/recorder/recorder.scenario.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ describe('recorder', () => {
await flushEvents()

expect(intakeRegistry.replaySegments.length).toBe(1)
const { segment, metadata, encoding, filename, mimetype } = intakeRegistry.replayRequests[0]
const {
segment,
metadata,
segmentFile: { encoding, filename, mimetype },
} = intakeRegistry.replayRequests[0]
expect(metadata).toEqual({
application: { id: jasmine.stringMatching(UUID_RE) },
creation_reason: 'init',
Expand Down
55 changes: 55 additions & 0 deletions test/e2e/scenario/transport.scenario.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { ExperimentalFeature } from '@datadog/browser-core'
import { createTest, flushEvents } from '../lib/framework'
import { getBrowserName, getPlatformName, withBrowserLogs } from '../lib/helpers/browser'

describe('transport', () => {
describe('data compression', () => {
createTest('send RUM data compressed')
.withRum({
enableExperimentalFeatures: [ExperimentalFeature.COMPRESS_BATCH],
})
.run(async ({ intakeRegistry }) => {
await flushEvents()

expect(intakeRegistry.rumRequests.length).toBe(2)

const plainRequest = intakeRegistry.rumRequests.find((request) => request.encoding === null)
const deflateRequest = intakeRegistry.rumRequests.find((request) => request.encoding === 'deflate')

// The last view update should be sent without compression
expect(plainRequest!.events).toEqual([
jasmine.objectContaining({
type: 'view',
}),
])

// Other data should be sent encoded
expect(deflateRequest!.events.length).toBeGreaterThan(0)
})

// Ignore this test on Safari desktop and Firefox because the Worker actually works even with
// CSP restriction.
// TODO: Remove this condition when upgrading to Safari 15 and Firefox 99
if (!((getBrowserName() === 'safari' && getPlatformName() === 'macos') || getBrowserName() === 'firefox')) {
Comment on lines +30 to +33
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💭 thought: ‏did you consider bumping them?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did, but I decided that it was not the right place to do it, as it might involve more things (everywhere we use getBrowserName for skipping tests, we should see if it still makes sense with newer versions)

createTest("displays a message if the worker can't be started")
.withRum({
enableExperimentalFeatures: [ExperimentalFeature.COMPRESS_BATCH],
})
.withBasePath('/no-blob-worker-csp')
.run(async ({ intakeRegistry }) => {
await flushEvents()

// Some non-deflate request can still be sent because on some browsers the Worker fails
// asynchronously
expect(intakeRegistry.rumRequests.filter((request) => request.encoding === 'deflate').length).toBe(0)

await withBrowserLogs((logs) => {
const failedToStartLog = logs.find((log) => log.message.includes('Datadog RUM failed to start'))
const cspDocLog = logs.find((log) => log.message.includes('Please make sure CSP'))
expect(failedToStartLog).withContext("'Failed to start' log").toBeTruthy()
expect(cspDocLog).withContext("'CSP doc' log").toBeTruthy()
})
})
}
})
})