Skip to content

Commit

Permalink
fix recording of webgl2 canvases (#106)
Browse files Browse the repository at this point in the history 
webgl2 canvases created after highlight would start recording with the
`preserveDrawingBuffer: false`
setting would not record correctly because we would snapshot a
transparent image of the canvas.
instead of trying to clear the canvas, we should create a context with
`preserveDrawingBuffer: true` to
ensure that the canvas can be recorded (tested in babylon.js)
https://app.highlight.io/1/sessions/n7P0x5XTItCqEN7B7pmtJVldUzJo
  • Loading branch information
@Vadman97
Vadman97 committed Jan 30, 2024
1 parent 66a4ce0 commit a85a66f
Show file tree
Hide file tree
Showing 5 changed files with 208 additions and 140 deletions.
2 changes: 2 additions & 0 deletions packages/rrweb/src/record/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ function record<T = eventWithTime>(
blockSelector,
mirror,
sampling: sampling?.canvas?.fps,
clearWebGLBuffer: sampling?.canvas?.clearWebGLBuffer,
initialSnapshotDelay: sampling?.canvas?.initialSnapshotDelay,
dataURLOptions,
resizeFactor: sampling?.canvas?.resizeFactor,
maxSnapshotDimension: sampling?.canvas?.maxSnapshotDimension,
Expand Down
303 changes: 170 additions & 133 deletions packages/rrweb/src/record/observers/canvas/canvas-manager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ export class CanvasManager {
blockSelector: string | null;
mirror: Mirror;
sampling?: 'all' | number;
clearWebGLBuffer?: boolean;
initialSnapshotDelay?: number;
dataURLOptions: DataURLOptions;
resizeFactor?: number;
maxSnapshotDimension?: number;
Expand All @@ -87,6 +89,8 @@ export class CanvasManager {
blockSelector,
recordCanvas,
recordVideos,
clearWebGLBuffer,
initialSnapshotDelay,
dataURLOptions,
} = options;
this.mutationCb = options.mutationCb;
Expand All @@ -103,6 +107,8 @@ export class CanvasManager {
blockClass,
blockSelector,
{
clearWebGLBuffer,
initialSnapshotDelay,
dataURLOptions,
},
options.resizeFactor,
Expand All @@ -111,15 +117,19 @@ export class CanvasManager {
}

private debug(
element: HTMLCanvasElement | HTMLVideoElement,
element: HTMLCanvasElement | HTMLVideoElement | null,
...args: Parameters<typeof console.log>
) {
if (!this.logger) return;
let prefix = `[highlight-${element.tagName.toLowerCase()}]`;
if (element.tagName.toLowerCase() === 'canvas') {
prefix += ` [ctx:${(element as ICanvas).__context}]`;
const id = this.mirror.getId(element);
let prefix = '[highlight-canvas-manager]';
if (element) {
prefix = `[highlight-${element.tagName.toLowerCase()}] [id:${id}]`;
if (element.tagName.toLowerCase() === 'canvas') {
prefix += ` [ctx:${(element as ICanvas).__context}]`;
}
}
this.logger.debug(prefix, element, ...args);
this.logger.debug(prefix, ...args);
}

private processMutation: canvasManagerMutationCallback = (
Expand All @@ -146,6 +156,8 @@ export class CanvasManager {
blockClass: blockClass,
blockSelector: string | null,
options: {
clearWebGLBuffer?: boolean;
initialSnapshotDelay?: number;
dataURLOptions: DataURLOptions;
},
resizeFactor?: number,
Expand All @@ -164,7 +176,12 @@ export class CanvasManager {
const { id } = e.data;
snapshotInProgressMap.set(id, false);

if (!('base64' in e.data)) return;
if (!('base64' in e.data)) {
this.debug(null, 'canvas worker received empty message', {
status: e.data.status,
});
return;
}

const { base64, type, dx, dy, dw, dh } = e.data;
this.mutationCb({
Expand Down Expand Up @@ -236,61 +253,73 @@ export class CanvasManager {
}
lastSnapshotTime = timestamp;

const promises: Promise<void>[] = []
promises.push(...getCanvas().map(async (canvas: HTMLCanvasElement) => {
this.debug(canvas, 'starting snapshotting');
const id = this.mirror.getId(canvas);
if (snapshotInProgressMap.get(id)) {
this.debug(canvas, 'snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
if (['webgl', 'webgl2'].includes((canvas as ICanvas).__context)) {
// if the canvas hasn't been modified recently,
// its contents won't be in memory and `createImageBitmap`
// will return a transparent imageBitmap

const context = canvas.getContext((canvas as ICanvas).__context) as
| WebGLRenderingContext
| WebGL2RenderingContext
| null;
const promises: Promise<void>[] = [];
promises.push(
...getCanvas().map(async (canvas: HTMLCanvasElement) => {
this.debug(canvas, 'starting snapshotting');
const id = this.mirror.getId(canvas);
if (snapshotInProgressMap.get(id)) {
this.debug(canvas, 'snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
if (
context?.getContextAttributes()?.preserveDrawingBuffer === false
options.clearWebGLBuffer !== false &&
['webgl', 'webgl2'].includes((canvas as ICanvas).__context)
) {
// Hack to load canvas back into memory so `createImageBitmap` can grab it's contents.
// Context: https://twitter.com/Juice10/status/1499775271758704643
// Preferably we set `preserveDrawingBuffer` to true, but that's not always possible,
// if the canvas hasn't been modified recently,
// its contents won't be in memory and `createImageBitmap`
// will return a transparent imageBitmap

const context = canvas.getContext(
(canvas as ICanvas).__context,
) as WebGLRenderingContext | WebGL2RenderingContext | null;
if (
context?.getContextAttributes()?.preserveDrawingBuffer === false
) {
// Hack to load canvas back into memory so `createImageBitmap` can grab it's contents.
// Context: https://twitter.com/Juice10/status/1499775271758704643
// Preferably we set `preserveDrawingBuffer` to true, but that's not always possible,
// especially when canvas is loaded before rrweb.
// This hack can wipe the background color of the canvas in the (unlikely) event that
// the canvas background was changed but clear was not called directly afterwards.
// the canvas background was changed but clear was not called directly afterwards.
// Example of this hack having negative side effect: https://visgl.github.io/react-map-gl/examples/layers
context.clear(context.COLOR_BUFFER_BIT);
context.clear(context?.COLOR_BUFFER_BIT);
this.debug(
canvas,
'cleared webgl canvas to load it into memory',
{ attributes: context?.getContextAttributes() },
);
}
}
}
// canvas is not yet ready... this retry on the next sampling iteration.
// we don't want to crash the worker if the canvas is not yet rendered.
if (canvas.width === 0 || canvas.height === 0) {
this.debug(canvas, 'not yet ready', {
width: canvas.width,
height: canvas.height,
// canvas is not yet ready... this retry on the next sampling iteration.
// we don't want to crash the worker by sending an undefined bitmap
// if the canvas is not yet rendered.
if (canvas.width === 0 || canvas.height === 0) {
this.debug(canvas, 'not yet ready', {
width: canvas.width,
height: canvas.height,
});
return;
}
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
const maxDim = Math.max(canvas.width, canvas.height);
scale = Math.min(scale, maxSnapshotDimension / maxDim);
}
const width = canvas.width * scale;
const height = canvas.height * scale;

const bitmap = await createImageBitmap(canvas, {
resizeWidth: width,
resizeHeight: height,
});
return;
}
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
const maxDim = Math.max(canvas.width, canvas.height);
scale = Math.min(scale, maxSnapshotDimension / maxDim);
}
const width = canvas.width * scale;
const height = canvas.height * scale;

const bitmap = await createImageBitmap(canvas, {
resizeWidth: width,
resizeHeight: height,
});
this.debug(canvas, 'created image bitmap');
worker.postMessage(
this.debug(canvas, 'created image bitmap', {
width: bitmap.width,
height: bitmap.height,
});
worker.postMessage(
{
id,
bitmap,
Expand All @@ -303,93 +332,101 @@ export class CanvasManager {
dataURLOptions: options.dataURLOptions,
},
[bitmap],
);
this.debug(canvas, 'sent message');
} catch (e) {
this.debug(canvas, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
}))
promises.push(...getVideos().map(async (video: HTMLVideoElement) => {
this.debug(video, 'starting video snapshotting');
const id = this.mirror.getId(video);
if (snapshotInProgressMap.get(id)) {
this.debug(video, 'video snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
const { width: boxWidth, height: boxHeight } =
video.getBoundingClientRect();
const { actualWidth, actualHeight } = {
actualWidth: video.videoWidth,
actualHeight: video.videoHeight,
};
const maxDim = Math.max(actualWidth, actualHeight);
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
scale = Math.min(scale, maxSnapshotDimension / maxDim);
);
this.debug(canvas, 'sent message');
} catch (e) {
this.debug(canvas, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
const width = actualWidth * scale;
const height = actualHeight * scale;

const bitmap = await createImageBitmap(video, {
resizeWidth: width,
resizeHeight: height,
});

let outputScale = Math.max(boxWidth, boxHeight) / maxDim;
const outputWidth = actualWidth * outputScale;
const outputHeight = actualHeight * outputScale;
const offsetX = (boxWidth - outputWidth) / 2;
const offsetY = (boxHeight - outputHeight) / 2;
this.debug(video, 'created image bitmap', {
actualWidth,
actualHeight,
boxWidth,
boxHeight,
outputWidth,
outputHeight,
resizeWidth: width,
resizeHeight: height,
scale,
outputScale,
offsetX,
offsetY,
});

worker.postMessage(
{
id,
bitmap,
width,
height,
dx: offsetX,
dy: offsetY,
dw: outputWidth,
dh: outputHeight,
dataURLOptions: options.dataURLOptions,
},
[bitmap],
);
this.debug(video, 'send message');
} catch (e) {
this.debug(video, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
}))
await Promise.all(promises)
}),
);
promises.push(
...getVideos().map(async (video: HTMLVideoElement) => {
this.debug(video, 'starting video snapshotting');
const id = this.mirror.getId(video);
if (snapshotInProgressMap.get(id)) {
this.debug(video, 'video snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
const { width: boxWidth, height: boxHeight } =
video.getBoundingClientRect();
const { actualWidth, actualHeight } = {
actualWidth: video.videoWidth,
actualHeight: video.videoHeight,
};
const maxDim = Math.max(actualWidth, actualHeight);
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
scale = Math.min(scale, maxSnapshotDimension / maxDim);
}
const width = actualWidth * scale;
const height = actualHeight * scale;

const bitmap = await createImageBitmap(video, {
resizeWidth: width,
resizeHeight: height,
});

let outputScale = Math.max(boxWidth, boxHeight) / maxDim;
const outputWidth = actualWidth * outputScale;
const outputHeight = actualHeight * outputScale;
const offsetX = (boxWidth - outputWidth) / 2;
const offsetY = (boxHeight - outputHeight) / 2;
this.debug(video, 'created image bitmap', {
actualWidth,
actualHeight,
boxWidth,
boxHeight,
outputWidth,
outputHeight,
resizeWidth: width,
resizeHeight: height,
scale,
outputScale,
offsetX,
offsetY,
});

worker.postMessage(
{
id,
bitmap,
width,
height,
dx: offsetX,
dy: offsetY,
dw: outputWidth,
dh: outputHeight,
dataURLOptions: options.dataURLOptions,
},
[bitmap],
);
this.debug(video, 'send message');
} catch (e) {
this.debug(video, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
}),
);
Promise.all(promises).catch(console.error);

rafId = requestAnimationFrame(takeSnapshots);
};

rafId = requestAnimationFrame(takeSnapshots);
const delay = setTimeout(() => {
rafId = requestAnimationFrame(takeSnapshots);
}, options.initialSnapshotDelay);

this.resetObservers = () => {
canvasContextReset();
cancelAnimationFrame(rafId);
clearTimeout(delay);
if (rafId) {
cancelAnimationFrame(rafId);
}
};
}

Expand Down
Loading

0 comments on commit a85a66f

Please sign in to comment.