From f1b23ddcccb1e1990b570abb8650afb95d4250db Mon Sep 17 00:00:00 2001 From: Yun Feng Date: Sun, 31 Jul 2022 09:01:04 +0800 Subject: [PATCH] fix: canvas data in iframe wasn't applied in the fast-forward mode (#944) * fix: canvas data in iframe wasn't applied in the fastforward mode * add more comments * Update packages/rrdom/src/diff.ts Co-authored-by: Justin Halsall * apply Juice10's suggestion Co-authored-by: Justin Halsall --- packages/rrdom/src/diff.ts | 19 +- packages/rrdom/src/index.ts | 1 + packages/rrweb-snapshot/src/rebuild.ts | 9 +- .../rrweb/test/events/canvas-in-iframe.ts | 181 ++++++++++++++++++ packages/rrweb/test/replayer.test.ts | 26 +++ 5 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 packages/rrweb/test/events/canvas-in-iframe.ts diff --git a/packages/rrdom/src/diff.ts b/packages/rrdom/src/diff.ts index b3a356d980..f0896088a9 100644 --- a/packages/rrdom/src/diff.ts +++ b/packages/rrdom/src/diff.ts @@ -136,14 +136,27 @@ export function diff( break; } case 'CANVAS': - (newTree as RRCanvasElement).canvasMutations.forEach( - (canvasMutation) => + { + const rrCanvasElement = newTree as RRCanvasElement; + // This canvas element is created with initial data in an iframe element. https://github.com/rrweb-io/rrweb/pull/944 + if (rrCanvasElement.rr_dataURL !== null) { + const image = document.createElement('img'); + image.onload = () => { + const ctx = (oldElement as HTMLCanvasElement).getContext('2d'); + if (ctx) { + ctx.drawImage(image, 0, 0, image.width, image.height); + } + }; + image.src = rrCanvasElement.rr_dataURL; + } + rrCanvasElement.canvasMutations.forEach((canvasMutation) => replayer.applyCanvas( canvasMutation.event, canvasMutation.mutation, oldTree as HTMLCanvasElement, ), - ); + ); + } break; case 'STYLE': applyVirtualStyleRulesToNode( diff --git a/packages/rrdom/src/index.ts b/packages/rrdom/src/index.ts index 646dc127ac..16da3ed8b9 100644 --- a/packages/rrdom/src/index.ts +++ b/packages/rrdom/src/index.ts @@ -149,6 +149,7 @@ export class RRElement extends BaseRRElementImpl(RRNode) { export class RRMediaElement extends BaseRRMediaElementImpl(RRElement) {} export class RRCanvasElement extends RRElement implements IRRElement { + public rr_dataURL: string | null = null; public canvasMutations: { event: canvasEventWithTime; mutation: canvasMutationData; diff --git a/packages/rrweb-snapshot/src/rebuild.ts b/packages/rrweb-snapshot/src/rebuild.ts index 7eebc90125..60e3669b11 100644 --- a/packages/rrweb-snapshot/src/rebuild.ts +++ b/packages/rrweb-snapshot/src/rebuild.ts @@ -228,13 +228,20 @@ function buildNode( // handle internal attributes if (tagName === 'canvas' && name === 'rr_dataURL') { const image = document.createElement('img'); - image.src = value; image.onload = () => { const ctx = (node as HTMLCanvasElement).getContext('2d'); if (ctx) { ctx.drawImage(image, 0, 0, image.width, image.height); } }; + image.src = value; + type RRCanvasElement = { + RRNodeType: NodeType; + rr_dataURL: string; + }; + // If the canvas element is created in RRDom runtime (seeking to a time point), the canvas context isn't supported. So the data has to be stored and not handled until diff process. https://github.com/rrweb-io/rrweb/pull/944 + if (((node as unknown) as RRCanvasElement).RRNodeType) + ((node as unknown) as RRCanvasElement).rr_dataURL = value; } else if (tagName === 'img' && name === 'rr_dataURL') { const image = node as HTMLImageElement; if (!image.currentSrc.startsWith('data:')) { diff --git a/packages/rrweb/test/events/canvas-in-iframe.ts b/packages/rrweb/test/events/canvas-in-iframe.ts new file mode 100644 index 0000000000..972d8ec13c --- /dev/null +++ b/packages/rrweb/test/events/canvas-in-iframe.ts @@ -0,0 +1,181 @@ +import { EventType, eventWithTime, IncrementalSource } from '../../src/types'; + +const now = Date.now(); + +const events: eventWithTime[] = [ + { + type: EventType.DomContentLoaded, + data: {}, + timestamp: now, + }, + { + type: EventType.Load, + data: {}, + timestamp: now + 100, + }, + { + type: EventType.Meta, + data: { + href: 'http://localhost', + width: 1200, + height: 500, + }, + timestamp: now + 100, + }, + { + type: EventType.FullSnapshot, + data: { + node: { + type: 0, + childNodes: [ + { + type: 2, + tagName: 'html', + attributes: {}, + childNodes: [ + { + type: 2, + tagName: 'head', + attributes: {}, + childNodes: [ + { type: 3, textContent: '\n ', id: 4 }, + { + type: 2, + tagName: 'meta', + attributes: { charset: 'utf-8' }, + childNodes: [], + id: 5, + }, + { type: 3, textContent: ' \n ', id: 6 }, + ], + id: 3, + }, + { type: 3, textContent: '\n ', id: 7 }, + { + type: 2, + tagName: 'body', + attributes: {}, + childNodes: [ + { type: 3, textContent: '\n ', id: 9 }, + { + type: 2, + tagName: 'iframe', + attributes: { id: 'target' }, + childNodes: [], + id: 19, + }, + { type: 3, textContent: '\n\n', id: 27 }, + ], + id: 8, + }, + ], + id: 2, + }, + ], + compatMode: 'BackCompat', + id: 1, + }, + initialOffset: { left: 0, top: 0 }, + }, + timestamp: now + 200, + }, + // add an iframe + { + type: EventType.IncrementalSnapshot, + data: { + source: IncrementalSource.Mutation, + adds: [ + { + parentId: 19, + nextId: null, + node: { + type: 0, + childNodes: [ + { + type: 2, + tagName: 'html', + attributes: {}, + childNodes: [ + { + type: 2, + tagName: 'head', + attributes: {}, + childNodes: [], + rootId: 30, + id: 32, + }, + { + type: 2, + tagName: 'body', + attributes: {}, + childNodes: [], + rootId: 30, + id: 33, + }, + ], + rootId: 30, + id: 31, + }, + ], + compatMode: 'BackCompat', + id: 30, + }, + }, + ], + removes: [], + texts: [], + attributes: [], + isAttachIframe: true, + }, + timestamp: now + 500, + }, + // add two canvas, one is blank ans the other is filled with data + { + type: EventType.IncrementalSnapshot, + data: { + source: 0, + texts: [], + attributes: [], + removes: [], + adds: [ + { + parentId: 33, + nextId: null, + node: { + type: 2, + tagName: 'canvas', + attributes: { + width: '10', + height: '10', + id: 'blank_canvas', + }, + childNodes: [], + rootId: 30, + id: 34, + }, + }, + { + parentId: 33, + nextId: null, + node: { + type: 2, + tagName: 'canvas', + attributes: { + width: '10', + height: '10', + rr_dataURL: + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAAXNSR0IArs4c6QAAAB5JREFUKFNjZCASMBKpjmEQKvzPwIDqrEHoRozgBQC/ZQELU4DiXAAAAABJRU5ErkJggg==', + id: 'canvas_with_data', + }, + childNodes: [], + rootId: 30, + id: 35, + }, + }, + ], + }, + timestamp: now + 500, + }, +]; + +export default events; diff --git a/packages/rrweb/test/replayer.test.ts b/packages/rrweb/test/replayer.test.ts index 4cfb3b0a31..01927e4dd1 100644 --- a/packages/rrweb/test/replayer.test.ts +++ b/packages/rrweb/test/replayer.test.ts @@ -15,6 +15,7 @@ import inputEvents from './events/input'; import iframeEvents from './events/iframe'; import shadowDomEvents from './events/shadow-dom'; import StyleSheetTextMutation from './events/style-sheet-text-mutation'; +import canvasInIframe from './events/canvas-in-iframe'; interface ISuite { code: string; @@ -613,6 +614,31 @@ describe('replayer', function () { ).toEqual('shadow dom two'); }); + it('can fast-forward mutation events containing painted canvas in iframe', async () => { + await page.evaluate(` + events = ${JSON.stringify(canvasInIframe)}; + const { Replayer } = rrweb; + var replayer = new Replayer(events,{showDebug:true}); + replayer.pause(550); + `); + const replayerIframe = await page.$('iframe'); + const contentDocument = await replayerIframe!.contentFrame()!; + const iframe = await contentDocument!.$('iframe'); + expect(iframe).not.toBeNull(); + const docInIFrame = await iframe?.contentFrame(); + expect(docInIFrame).not.toBeNull(); + const canvasElements = await docInIFrame!.$$('canvas'); + // The first canvas is a blank one and the second is a painted one. + expect(canvasElements.length).toEqual(2); + + const dataUrls = await docInIFrame?.$$eval('canvas', (elements) => + elements.map((element) => (element as HTMLCanvasElement).toDataURL()), + ); + expect(dataUrls?.length).toEqual(2); + // The painted canvas's data should not be empty. + expect(dataUrls![1]).not.toEqual(dataUrls![0]); + }); + it('can stream events in live mode', async () => { const status = await page.evaluate(` const { Replayer } = rrweb;