From 69263d588253c973553002b0e4771fe222aa3dcb Mon Sep 17 00:00:00 2001 From: Vadim Korolik Date: Fri, 21 Jul 2023 14:10:38 -0700 Subject: [PATCH] fix recording of webgl2 canvases (#106) webgl2 canvases created after highlight would start recording with the `preserveDrawingBuffer: false` setting would not record correctly because we would snapshot a transparent image of the canvas. instead of trying to clear the canvas, we should create a context with `preserveDrawingBuffer: true` to ensure that the canvas can be recorded (tested in babylon.js) https://app.highlight.io/1/sessions/n7P0x5XTItCqEN7B7pmtJVldUzJo --- packages/rrweb/src/record/index.ts | 2 + .../record/observers/canvas/canvas-manager.ts | 303 ++++++++++-------- .../src/record/observers/canvas/canvas.ts | 15 +- .../workers/image-bitmap-data-url-worker.ts | 24 +- packages/types/src/index.ts | 11 + 5 files changed, 213 insertions(+), 142 deletions(-) diff --git a/packages/rrweb/src/record/index.ts b/packages/rrweb/src/record/index.ts index 7e9bb55b..b5b1ceb2 100644 --- a/packages/rrweb/src/record/index.ts +++ b/packages/rrweb/src/record/index.ts @@ -320,6 +320,8 @@ function record( blockSelector, mirror, sampling: sampling?.canvas?.fps, + clearWebGLBuffer: sampling?.canvas?.clearWebGLBuffer, + initialSnapshotDelay: sampling?.canvas?.initialSnapshotDelay, dataURLOptions, resizeFactor: sampling?.canvas?.resizeFactor, maxSnapshotDimension: sampling?.canvas?.maxSnapshotDimension, diff --git a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts index a93d3505..36815dc1 100644 --- a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts +++ b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts @@ -72,6 +72,8 @@ export class CanvasManager { blockSelector: string | null; mirror: Mirror; sampling?: 'all' | number; + clearWebGLBuffer?: boolean; + initialSnapshotDelay?: number; dataURLOptions: DataURLOptions; resizeFactor?: number; maxSnapshotDimension?: number; @@ -87,6 +89,8 @@ export class CanvasManager { blockSelector, recordCanvas, recordVideos, + clearWebGLBuffer, + initialSnapshotDelay, dataURLOptions, } = options; this.mutationCb = options.mutationCb; @@ -103,6 +107,8 @@ export class CanvasManager { blockClass, blockSelector, { + clearWebGLBuffer, + initialSnapshotDelay, dataURLOptions, }, options.resizeFactor, @@ -111,15 +117,19 @@ export class CanvasManager { } private debug( - element: HTMLCanvasElement | HTMLVideoElement, + element: HTMLCanvasElement | HTMLVideoElement | null, ...args: Parameters ) { if (!this.logger) return; - let prefix = `[highlight-${element.tagName.toLowerCase()}]`; - if (element.tagName.toLowerCase() === 'canvas') { - prefix += ` [ctx:${(element as ICanvas).__context}]`; + const id = this.mirror.getId(element); + let prefix = '[highlight-canvas-manager]'; + if (element) { + prefix = `[highlight-${element.tagName.toLowerCase()}] [id:${id}]`; + if (element.tagName.toLowerCase() === 'canvas') { + prefix += ` [ctx:${(element as ICanvas).__context}]`; + } } - this.logger.debug(prefix, element, ...args); + this.logger.debug(prefix, ...args); } private processMutation: canvasManagerMutationCallback = ( @@ -146,6 +156,8 @@ export class CanvasManager { blockClass: blockClass, blockSelector: string | null, options: { + clearWebGLBuffer?: boolean; + initialSnapshotDelay?: number; dataURLOptions: DataURLOptions; }, resizeFactor?: number, @@ -163,7 +175,12 @@ export class CanvasManager { const { id } = e.data; snapshotInProgressMap.set(id, false); - if (!('base64' in e.data)) return; + if (!('base64' in e.data)) { + this.debug(null, 'canvas worker received empty message', { + status: e.data.status, + }); + return; + } const { base64, type, dx, dy, dw, dh } = e.data; this.mutationCb({ @@ -235,58 +252,70 @@ export class CanvasManager { } lastSnapshotTime = timestamp; - const promises: Promise[] = [] - promises.push(...getCanvas().map(async (canvas: HTMLCanvasElement) => { - this.debug(canvas, 'starting snapshotting'); - const id = this.mirror.getId(canvas); - if (snapshotInProgressMap.get(id)) { - this.debug(canvas, 'snapshotting already in progress for', id); - return; - } - snapshotInProgressMap.set(id, true); - try { - if (['webgl', 'webgl2'].includes((canvas as ICanvas).__context)) { - // if the canvas hasn't been modified recently, - // its contents won't be in memory and `createImageBitmap` - // will return a transparent imageBitmap - - const context = canvas.getContext((canvas as ICanvas).__context) as - | WebGLRenderingContext - | WebGL2RenderingContext - | null; + const promises: Promise[] = []; + promises.push( + ...getCanvas().map(async (canvas: HTMLCanvasElement) => { + this.debug(canvas, 'starting snapshotting'); + const id = this.mirror.getId(canvas); + if (snapshotInProgressMap.get(id)) { + this.debug(canvas, 'snapshotting already in progress for', id); + return; + } + snapshotInProgressMap.set(id, true); + try { if ( - context?.getContextAttributes()?.preserveDrawingBuffer === false + options.clearWebGLBuffer !== false && + ['webgl', 'webgl2'].includes((canvas as ICanvas).__context) ) { - // Hack to load canvas back into memory so `createImageBitmap` can grab it's contents. - // Context: https://twitter.com/Juice10/status/1499775271758704643 - // This hack might change the background color of the canvas in the unlikely event that - // the canvas background was changed but clear was not called directly afterwards. - context?.clear(context.COLOR_BUFFER_BIT); + // if the canvas hasn't been modified recently, + // its contents won't be in memory and `createImageBitmap` + // will return a transparent imageBitmap + + const context = canvas.getContext( + (canvas as ICanvas).__context, + ) as WebGLRenderingContext | WebGL2RenderingContext | null; + if ( + context?.getContextAttributes()?.preserveDrawingBuffer === false + ) { + // Hack to load canvas back into memory so `createImageBitmap` can grab it's contents. + // Context: https://twitter.com/Juice10/status/1499775271758704643 + // This hack might change the background color of the canvas in the unlikely event that + // the canvas background was changed but clear was not called directly afterwards. + context?.clear(context?.COLOR_BUFFER_BIT); + this.debug( + canvas, + 'cleared webgl canvas to load it into memory', + { attributes: context?.getContextAttributes() }, + ); + } } - } - // canvas is not yet ready... this retry on the next sampling iteration. - // we don't want to crash the worker if the canvas is not yet rendered. - if (canvas.width === 0 || canvas.height === 0) { - this.debug(canvas, 'not yet ready', { - width: canvas.width, - height: canvas.height, + // canvas is not yet ready... this retry on the next sampling iteration. + // we don't want to crash the worker by sending an undefined bitmap + // if the canvas is not yet rendered. + if (canvas.width === 0 || canvas.height === 0) { + this.debug(canvas, 'not yet ready', { + width: canvas.width, + height: canvas.height, + }); + return; + } + let scale = resizeFactor || 1; + if (maxSnapshotDimension) { + const maxDim = Math.max(canvas.width, canvas.height); + scale = Math.min(scale, maxSnapshotDimension / maxDim); + } + const width = canvas.width * scale; + const height = canvas.height * scale; + + const bitmap = await createImageBitmap(canvas, { + resizeWidth: width, + resizeHeight: height, }); - return; - } - let scale = resizeFactor || 1; - if (maxSnapshotDimension) { - const maxDim = Math.max(canvas.width, canvas.height); - scale = Math.min(scale, maxSnapshotDimension / maxDim); - } - const width = canvas.width * scale; - const height = canvas.height * scale; - - const bitmap = await createImageBitmap(canvas, { - resizeWidth: width, - resizeHeight: height, - }); - this.debug(canvas, 'created image bitmap'); - worker.postMessage( + this.debug(canvas, 'created image bitmap', { + width: bitmap.width, + height: bitmap.height, + }); + worker.postMessage( { id, bitmap, @@ -299,93 +328,101 @@ export class CanvasManager { dataURLOptions: options.dataURLOptions, }, [bitmap], - ); - this.debug(canvas, 'sent message'); - } catch (e) { - this.debug(canvas, 'failed to snapshot', e); - } finally { - snapshotInProgressMap.set(id, false); - } - })) - promises.push(...getVideos().map(async (video: HTMLVideoElement) => { - this.debug(video, 'starting video snapshotting'); - const id = this.mirror.getId(video); - if (snapshotInProgressMap.get(id)) { - this.debug(video, 'video snapshotting already in progress for', id); - return; - } - snapshotInProgressMap.set(id, true); - try { - const { width: boxWidth, height: boxHeight } = - video.getBoundingClientRect(); - const { actualWidth, actualHeight } = { - actualWidth: video.videoWidth, - actualHeight: video.videoHeight, - }; - const maxDim = Math.max(actualWidth, actualHeight); - let scale = resizeFactor || 1; - if (maxSnapshotDimension) { - scale = Math.min(scale, maxSnapshotDimension / maxDim); + ); + this.debug(canvas, 'sent message'); + } catch (e) { + this.debug(canvas, 'failed to snapshot', e); + } finally { + snapshotInProgressMap.set(id, false); } - const width = actualWidth * scale; - const height = actualHeight * scale; - - const bitmap = await createImageBitmap(video, { - resizeWidth: width, - resizeHeight: height, - }); - - let outputScale = Math.max(boxWidth, boxHeight) / maxDim; - const outputWidth = actualWidth * outputScale; - const outputHeight = actualHeight * outputScale; - const offsetX = (boxWidth - outputWidth) / 2; - const offsetY = (boxHeight - outputHeight) / 2; - this.debug(video, 'created image bitmap', { - actualWidth, - actualHeight, - boxWidth, - boxHeight, - outputWidth, - outputHeight, - resizeWidth: width, - resizeHeight: height, - scale, - outputScale, - offsetX, - offsetY, - }); - - worker.postMessage( - { - id, - bitmap, - width, - height, - dx: offsetX, - dy: offsetY, - dw: outputWidth, - dh: outputHeight, - dataURLOptions: options.dataURLOptions, - }, - [bitmap], - ); - this.debug(video, 'send message'); - } catch (e) { - this.debug(video, 'failed to snapshot', e); - } finally { - snapshotInProgressMap.set(id, false); - } - })) - await Promise.all(promises) + }), + ); + promises.push( + ...getVideos().map(async (video: HTMLVideoElement) => { + this.debug(video, 'starting video snapshotting'); + const id = this.mirror.getId(video); + if (snapshotInProgressMap.get(id)) { + this.debug(video, 'video snapshotting already in progress for', id); + return; + } + snapshotInProgressMap.set(id, true); + try { + const { width: boxWidth, height: boxHeight } = + video.getBoundingClientRect(); + const { actualWidth, actualHeight } = { + actualWidth: video.videoWidth, + actualHeight: video.videoHeight, + }; + const maxDim = Math.max(actualWidth, actualHeight); + let scale = resizeFactor || 1; + if (maxSnapshotDimension) { + scale = Math.min(scale, maxSnapshotDimension / maxDim); + } + const width = actualWidth * scale; + const height = actualHeight * scale; + + const bitmap = await createImageBitmap(video, { + resizeWidth: width, + resizeHeight: height, + }); + + let outputScale = Math.max(boxWidth, boxHeight) / maxDim; + const outputWidth = actualWidth * outputScale; + const outputHeight = actualHeight * outputScale; + const offsetX = (boxWidth - outputWidth) / 2; + const offsetY = (boxHeight - outputHeight) / 2; + this.debug(video, 'created image bitmap', { + actualWidth, + actualHeight, + boxWidth, + boxHeight, + outputWidth, + outputHeight, + resizeWidth: width, + resizeHeight: height, + scale, + outputScale, + offsetX, + offsetY, + }); + + worker.postMessage( + { + id, + bitmap, + width, + height, + dx: offsetX, + dy: offsetY, + dw: outputWidth, + dh: outputHeight, + dataURLOptions: options.dataURLOptions, + }, + [bitmap], + ); + this.debug(video, 'send message'); + } catch (e) { + this.debug(video, 'failed to snapshot', e); + } finally { + snapshotInProgressMap.set(id, false); + } + }), + ); + Promise.all(promises).catch(console.error); rafId = requestAnimationFrame(takeSnapshots); }; - rafId = requestAnimationFrame(takeSnapshots); + const delay = setTimeout(() => { + rafId = requestAnimationFrame(takeSnapshots); + }, options.initialSnapshotDelay); this.resetObservers = () => { canvasContextReset(); - cancelAnimationFrame(rafId); + clearTimeout(delay); + if (rafId) { + cancelAnimationFrame(rafId); + } }; } diff --git a/packages/rrweb/src/record/observers/canvas/canvas.ts b/packages/rrweb/src/record/observers/canvas/canvas.ts index 87d9c5cb..4eb1ede8 100644 --- a/packages/rrweb/src/record/observers/canvas/canvas.ts +++ b/packages/rrweb/src/record/observers/canvas/canvas.ts @@ -13,7 +13,7 @@ export default function initCanvasContextObserver( ): listenerHandler { const handlers: listenerHandler[] = []; try { - const restoreHandler = patch( + const restoreGetContext = patch( win.HTMLCanvasElement.prototype, 'getContext', function ( @@ -21,21 +21,24 @@ export default function initCanvasContextObserver( this: ICanvas, contextType: string, ...args: Array - ) => void, + ) => unknown, ) { return function ( this: ICanvas, contextType: string, ...args: Array ) { - if (!isBlocked(this, blockClass, blockSelector, true)) { - if (!this.__context) this.__context = contextType; + const ctx = original.apply(this, [contextType, ...args]); + if (ctx) { + if (!isBlocked(this, blockClass, blockSelector, true)) { + if (!this.__context) this.__context = contextType; + } } - return original.apply(this, [contextType, ...args]); + return ctx; }; }, ); - handlers.push(restoreHandler); + handlers.push(restoreGetContext); } catch { console.error('failed to patch HTMLCanvasElement.prototype.getContext'); } diff --git a/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts b/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts index cdafa900..07e2f443 100644 --- a/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts +++ b/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts @@ -71,11 +71,26 @@ worker.onmessage = async function (e) { // on first try we should check if canvas is transparent, // no need to save it's contents in that case if (!lastBlobMap.has(id) && (await transparentBase64) === base64) { + console.debug('[highlight-worker] canvas bitmap is transparent', { + id, + base64, + }); lastBlobMap.set(id, base64); - return worker.postMessage({ id }); + return worker.postMessage({ id, status: 'transparent' }); } - if (lastBlobMap.get(id) === base64) return worker.postMessage({ id }); // unchanged + // unchanged + if (lastBlobMap.get(id) === base64) { + console.debug('[highlight-worker] canvas bitmap is unchanged', { + id, + base64, + }); + return worker.postMessage({ id, status: 'unchanged' }); + } + console.debug('[highlight-worker] canvas bitmap processed', { + id, + base64, + }); worker.postMessage({ id, type, @@ -89,6 +104,9 @@ worker.onmessage = async function (e) { }); lastBlobMap.set(id, base64); } else { - return worker.postMessage({ id: e.data.id }); + console.debug('[highlight-worker] no offscreencanvas support', { + id: e.data.id, + }); + return worker.postMessage({ id: e.data.id, status: 'unsupported' }); } }; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index dad87168..96344c65 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -207,6 +207,16 @@ export type CanvasSamplingStrategy = Partial<{ * in either dimension (while preserving the original canvas aspect ratio). */ maxSnapshotDimension: number; + /** + * Default behavior for WebGL canvas elements with `preserveDrawingBuffer: false` is to clear the buffer to + * load the canvas into memory to avoid getting a transparent bitmap. + * Set to false to disable the clearing (in case there are visual glitches in the canvas). + */ + clearWebGLBuffer?: boolean; + /** + * Time (in milliseconds) to wait before the initial snapshot of canvas/video elements. + */ + initialSnapshotDelay?: number; /** * Adjust the quality of the canvas blob serialization. */ @@ -542,6 +552,7 @@ export type ImageBitmapDataURLWorkerParams = { export type ImageBitmapDataURLWorkerResponse = | { id: number; + status: string; } | { id: number;