record canvas mutations

close #60, #261

This patch implements the canvas mutation observer.
It consists of both the record and the replay side changes.

In the record side, we add a `recordCanvas` flag to indicate
whether to record canvas elements and the flag defaults to false.
Different from our other observers, the canvas observer was
disabled by default. Because some applications with heavy canvas
usage may emit a lot of data as canvas changed, especially the
scenarios that use a lot of `drawImage` API.
So the behavior should be audited by users and only record canvas
when the flag was set to true.

In the replay side, we add a `UNSAFE_replayCanvas` flag to indicate
whether to replay canvas mutations.
Similar to the `recordCanvas` flag, `UNSAFE_replayCanvas` defaults
to false. But unlike the record canvas implementation is stable and
safe, the replay canvas implementation is UNSAFE.
It's unsafe because we need to add `allow-scripts` to the replay
sandbox, which may cause some unexpected script execution. Currently,
users should be aware of this implementation detail and enable this
feature carefully.

package.json

@@ -62,7 +62,7 @@
     "@xstate/fsm": "^1.4.0",
     "mitt": "^1.1.3",
     "pako": "^1.0.11",
-    "rrweb-snapshot": "^0.8.0",
+    "rrweb-snapshot": "^0.8.1",
     "smoothscroll-polyfill": "^0.4.3"
   }
 }

scripts/repl.ts

@@ -81,7 +81,7 @@ function getCode(): string {
         width: 1600,
         height: 900,
       },
-      args: ['--start-maximized'],
+      args: ['--start-maximized', '--ignore-certificate-errors'],
     });
     const page = await browser.newPage();
     await page.goto(url, {
@@ -94,7 +94,8 @@ function getCode(): string {
     await page.evaluate(`;${code}
       window.__IS_RECORDING__ = true
       rrweb.record({
-        emit: event => window._replLog(event)
+        emit: event => window._replLog(event),
+        recordCanvas: true
       });
     `);
     page.on('framenavigated', async () => {
@@ -103,14 +104,14 @@ function getCode(): string {
         await page.evaluate(`;${code}
           window.__IS_RECORDING__ = true
           rrweb.record({
-            emit: event => window._replLog(event)
+            emit: event => window._replLog(event),
+            recordCanvas: true
           });
         `);
       }
     });
 
-    emitter.once('done', async shouldReplay => {
-      console.log(`Recorded ${events.length} events`);
+    emitter.once('done', async (shouldReplay) => {
       await browser.close();
       if (shouldReplay) {
         await replay();
@@ -170,7 +171,9 @@ function getCode(): string {
         '<\\/script>',
       )};
       /*-->*/
-      const replayer = new rrweb.Replayer(events);
+      const replayer = new rrweb.Replayer(events, {
+        UNSAFE_replayCanvas: true
+      });
       replayer.play();
     </script>
   </body>
@@ -182,10 +185,10 @@ function getCode(): string {
   }
 
   process
-    .on('uncaughtException', error => {
+    .on('uncaughtException', (error) => {
       console.error(error);
     })
-    .on('unhandledRejection', error => {
+    .on('unhandledRejection', (error) => {
       console.error(error);
     });
 })();

src/record/index.ts

@@ -41,6 +41,7 @@ function record<T = eventWithTime>(
     packFn,
     sampling = {},
     mousemoveWait,
+    recordCanvas = false,
   } = options;
   // runtime checks for user options
   if (!emit) {
@@ -113,6 +114,7 @@ function record<T = eventWithTime>(
       blockClass,
       inlineStylesheet,
       maskInputOptions,
+      recordCanvas,
     );
 
     if (!node) {
@@ -244,11 +246,22 @@ function record<T = eventWithTime>(
                   },
                 }),
               ),
+            canvasMutationCb: (p) =>
+              wrappedEmit(
+                wrapEvent({
+                  type: EventType.IncrementalSnapshot,
+                  data: {
+                    source: IncrementalSource.CanvasMutation,
+                    ...p,
+                  },
+                }),
+              ),
             blockClass,
             ignoreClass,
             maskInputOptions,
             inlineStylesheet,
             sampling,
+            recordCanvas,
           },
           hooks,
         ),

src/record/mutation.ts

@@ -138,16 +138,19 @@ export default class MutationBuffer {
   private blockClass: blockClass;
   private inlineStylesheet: boolean;
   private maskInputOptions: MaskInputOptions;
+  private recordCanvas: boolean;
 
   constructor(
     cb: mutationCallBack,
     blockClass: blockClass,
     inlineStylesheet: boolean,
     maskInputOptions: MaskInputOptions,
+    recordCanvas: boolean,
   ) {
     this.blockClass = blockClass;
     this.inlineStylesheet = inlineStylesheet;
     this.maskInputOptions = maskInputOptions;
+    this.recordCanvas = recordCanvas;
     this.emissionCallback = cb;
   }
 
@@ -187,6 +190,7 @@ export default class MutationBuffer {
           true,
           this.inlineStylesheet,
           this.maskInputOptions,
+          this.recordCanvas,
         )!,
       });
     };

src/record/observer.ts

@@ -8,6 +8,7 @@ import {
   getWindowWidth,
   isBlocked,
   isTouchEvent,
+  patch,
 } from '../utils';
 import {
   mutationCallBack,
@@ -30,6 +31,7 @@ import {
   mediaInteractionCallback,
   MediaInteractions,
   SamplingStrategy,
+  canvasMutationCallback,
 } from '../types';
 import MutationBuffer from './mutation';
 
@@ -38,13 +40,15 @@ function initMutationObserver(
   blockClass: blockClass,
   inlineStylesheet: boolean,
   maskInputOptions: MaskInputOptions,
+  recordCanvas: boolean,
 ): MutationObserver {
   // see mutation.ts for details
   const mutationBuffer = new MutationBuffer(
     cb,
     blockClass,
     inlineStylesheet,
     maskInputOptions,
+    recordCanvas,
   );
   const observer = new MutationObserver(mutationBuffer.processMutations);
   observer.observe(document, {
@@ -357,6 +361,75 @@ function initMediaInteractionObserver(
   };
 }
 
+function initCanvasMutationObserver(
+  cb: canvasMutationCallback,
+  blockClass: blockClass,
+): listenerHandler {
+  const props = Object.getOwnPropertyNames(CanvasRenderingContext2D.prototype);
+  const handlers: listenerHandler[] = [];
+  for (const prop of props) {
+    try {
+      if (
+        typeof CanvasRenderingContext2D.prototype[
+          prop as keyof CanvasRenderingContext2D
+        ] !== 'function'
+      ) {
+        continue;
+      }
+      const restoreHandler = patch(
+        CanvasRenderingContext2D.prototype,
+        prop,
+        function (original) {
+          return function (
+            this: CanvasRenderingContext2D,
+            ...args: Array<unknown>
+          ) {
+            if (!isBlocked(this.canvas, blockClass)) {
+              setTimeout(() => {
+                const recordArgs = [...args];
+                if (prop === 'drawImage') {
+                  if (
+                    recordArgs[0] &&
+                    recordArgs[0] instanceof HTMLCanvasElement
+                  ) {
+                    recordArgs[0] = recordArgs[0].toDataURL();
+                  }
+                }
+                cb({
+                  id: mirror.getId((this.canvas as unknown) as INode),
+                  property: prop,
+                  args: recordArgs,
+                });
+              }, 0);
+            }
+            return original.apply(this, args);
+          };
+        },
+      );
+      handlers.push(restoreHandler);
+    } catch {
+      const hookHandler = hookSetter<CanvasRenderingContext2D>(
+        CanvasRenderingContext2D.prototype,
+        prop,
+        {
+          set(v) {
+            cb({
+              id: mirror.getId((this.canvas as unknown) as INode),
+              property: prop,
+              args: [v],
+              setter: true,
+            });
+          },
+        },
+      );
+      handlers.push(hookHandler);
+    }
+  }
+  return () => {
+    handlers.forEach((h) => h());
+  };
+}
+
 function mergeHooks(o: observerParam, hooks: hooksParam) {
   const {
     mutationCb,
@@ -367,6 +440,7 @@ function mergeHooks(o: observerParam, hooks: hooksParam) {
     inputCb,
     mediaInteractionCb,
     styleSheetRuleCb,
+    canvasMutationCb,
   } = o;
   o.mutationCb = (...p: Arguments<mutationCallBack>) => {
     if (hooks.mutation) {
@@ -416,6 +490,12 @@ function mergeHooks(o: observerParam, hooks: hooksParam) {
     }
     styleSheetRuleCb(...p);
   };
+  o.canvasMutationCb = (...p: Arguments<canvasMutationCallback>) => {
+    if (hooks.canvasMutation) {
+      hooks.canvasMutation(...p);
+    }
+    canvasMutationCb(...p);
+  };
 }
 
 export default function initObservers(
@@ -428,6 +508,7 @@ export default function initObservers(
     o.blockClass,
     o.inlineStylesheet,
     o.maskInputOptions,
+    o.recordCanvas,
   );
   const mousemoveHandler = initMoveObserver(o.mousemoveCb, o.sampling);
   const mouseInteractionHandler = initMouseInteractionObserver(
@@ -453,6 +534,9 @@ export default function initObservers(
     o.blockClass,
   );
   const styleSheetObserver = initStyleSheetObserver(o.styleSheetRuleCb);
+  const canvasMutationObserver = o.recordCanvas
+    ? initCanvasMutationObserver(o.canvasMutationCb, o.blockClass)
+    : () => {};
 
   return () => {
     mutationObserver.disconnect();
@@ -463,5 +547,6 @@ export default function initObservers(
     inputHandler();
     mediaInteractionHandler();
     styleSheetObserver();
+    canvasMutationObserver();
   };
 }