Fix Raster DEM decoding in safari private browsing mode (#3185)

* working * tests * rm force * rm debug * convert promise to callback and some tricks to reduce bundle size * fix test * back to async/await * mangle test * indent * fix mangle test * update changelog * changelog tweak * respond to comments * fixes * changelog * comments * comment * fix test * handle html image decoding in main thread * tweak * bump size * comments * refactor getImage call
maplibre · Oct 10, 2023 · 8edef2b · 8edef2b
1 parent be14c41
commit 8edef2b
Show file tree

Hide file tree

Showing 8 changed files with 404 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,7 +6,8 @@
 
 ### 🐞 Bug fixes
 
-- Fix setStyle->style.setState didn't reset _serializedLayers ([#3133](https://github.com/maplibre/maplibre-gl-js/pull/3133)).
+- Fix setStyle->style.setState didn't reset \_serializedLayers ([#3133](https://github.com/maplibre/maplibre-gl-js/pull/3133)).
+- Fix Raster DEM decoding in safari private browsing mode ([#3185](https://github.com/maplibre/maplibre-gl-js/pull/3185))
 - _...Add new stuff here..._
 
 ## 3.4.0

diff --git a/src/source/raster_dem_tile_source.ts b/src/source/raster_dem_tile_source.ts
@@ -1,6 +1,6 @@
 import {ImageRequest} from '../util/image_request';
 import {ResourceType} from '../util/request_manager';
-import {extend, isImageBitmap} from '../util/util';
+import {extend, isImageBitmap, readImageUsingVideoFrame} from '../util/util';
 import {Evented} from '../util/evented';
 import {browser} from '../util/browser';
 import {offscreenCanvasSupported} from '../util/offscreen_canvas_supported';
@@ -16,6 +16,8 @@ import type {Tile} from './tile';
 import type {Callback} from '../types/callback';
 import type {RasterDEMSourceSpecification} from '@maplibre/maplibre-gl-style-spec';
 import type {ExpiryData} from '../util/ajax';
+import {isOffscreenCanvasDistorted} from '../util/offscreen_canvas_distorted';
+import {RGBAImage} from '../util/image';
 
 /**
  * A source containing raster DEM tiles (See the [Style Specification](https://maplibre.org/maplibre-style-spec/) for detailed documentation of options.)
@@ -54,10 +56,9 @@ export class RasterDEMTileSource extends RasterTileSource implements Source {
 
     loadTile(tile: Tile, callback: Callback<void>) {
         const url = tile.tileID.canonical.url(this.tiles, this.map.getPixelRatio(), this.scheme);
-        tile.request = ImageRequest.getImage(this.map._requestManager.transformRequest(url, ResourceType.Tile), imageLoaded.bind(this), this.map._refreshExpiredTiles);
-
+        const request = this.map._requestManager.transformRequest(url, ResourceType.Tile);
         tile.neighboringTiles = this._getNeighboringTiles(tile.tileID);
-        function imageLoaded(err: Error, img: (HTMLImageElement | ImageBitmap) & ExpiryData) {
+        tile.request = ImageRequest.getImage(request, async (err: Error, img: (HTMLImageElement | ImageBitmap), expiry: ExpiryData) => {
             delete tile.request;
             if (tile.aborted) {
                 tile.state = 'unloaded';
@@ -66,11 +67,9 @@ export class RasterDEMTileSource extends RasterTileSource implements Source {
                 tile.state = 'errored';
                 callback(err);
             } else if (img) {
-                if (this.map._refreshExpiredTiles) tile.setExpiryData(img);
-                delete img.cacheControl;
-                delete img.expires;
+                if (this.map._refreshExpiredTiles) tile.setExpiryData(expiry);
                 const transfer = isImageBitmap(img) && offscreenCanvasSupported();
-                const rawImageData = transfer ? img : browser.getImageData(img, 1);
+                const rawImageData = transfer ? img : await readImageNow(img);
                 const params = {
                     uid: tile.uid,
                     coord: tile.tileID,
@@ -85,9 +84,22 @@ export class RasterDEMTileSource extends RasterTileSource implements Source {
 
                 if (!tile.actor || tile.state === 'expired') {
                     tile.actor = this.dispatcher.getActor();
-                    tile.actor.send('loadDEMTile', params, done.bind(this));
+                    tile.actor.send('loadDEMTile', params, done);
+                }
+            }
+        }, this.map._refreshExpiredTiles);
+
+        async function readImageNow(img: ImageBitmap | HTMLImageElement): Promise<RGBAImage | ImageData> {
+            if (typeof VideoFrame !== 'undefined' && isOffscreenCanvasDistorted()) {
+                const width = img.width + 2;
+                const height = img.height + 2;
+                try {
+                    return new RGBAImage({width, height}, await readImageUsingVideoFrame(img, -1, -1, width, height));
+                } catch (e) {
+                    // fall-back to browser canvas decoding
                 }
             }
+            return browser.getImageData(img, 1);
         }
 
         function done(err, data) {

diff --git a/src/source/raster_dem_tile_worker_source.ts b/src/source/raster_dem_tile_worker_source.ts
@@ -6,46 +6,29 @@ import type {
     WorkerDEMTileCallback,
     TileParameters
 } from './worker_source';
-import {isImageBitmap} from '../util/util';
+import {getImageData, isImageBitmap} from '../util/util';
 
 export class RasterDEMTileWorkerSource {
     actor: Actor;
     loaded: {[_: string]: DEMData};
-    offscreenCanvas: OffscreenCanvas;
-    offscreenCanvasContext: OffscreenCanvasRenderingContext2D;
 
     constructor() {
         this.loaded = {};
     }
 
-    loadTile(params: WorkerDEMTileParameters, callback: WorkerDEMTileCallback) {
+    async loadTile(params: WorkerDEMTileParameters, callback: WorkerDEMTileCallback) {
         const {uid, encoding, rawImageData, redFactor, greenFactor, blueFactor, baseShift} = params;
-        // Main thread will transfer ImageBitmap if offscreen decode with OffscreenCanvas is supported, else it will transfer an already decoded image.
-        const imagePixels = isImageBitmap(rawImageData) ? this.getImageData(rawImageData) : rawImageData as RGBAImage;
+        const width = rawImageData.width + 2;
+        const height = rawImageData.height + 2;
+        const imagePixels: RGBAImage = isImageBitmap(rawImageData) ?
+            new RGBAImage({width, height}, await getImageData(rawImageData, -1, -1, width, height)) :
+            rawImageData;
         const dem = new DEMData(uid, imagePixels, encoding, redFactor, greenFactor, blueFactor, baseShift);
         this.loaded = this.loaded || {};
         this.loaded[uid] = dem;
         callback(null, dem);
     }
 
-    getImageData(imgBitmap: ImageBitmap): RGBAImage {
-        // Lazily initialize OffscreenCanvas
-        if (!this.offscreenCanvas || !this.offscreenCanvasContext) {
-            // Dem tiles are typically 256x256
-            this.offscreenCanvas = new OffscreenCanvas(imgBitmap.width, imgBitmap.height);
-            this.offscreenCanvasContext = this.offscreenCanvas.getContext('2d', {willReadFrequently: true});
-        }
-
-        this.offscreenCanvas.width = imgBitmap.width;
-        this.offscreenCanvas.height = imgBitmap.height;
-
-        this.offscreenCanvasContext.drawImage(imgBitmap, 0, 0, imgBitmap.width, imgBitmap.height);
-        // Insert an additional 1px padding around the image to allow backfilling for neighboring data.
-        const imgData = this.offscreenCanvasContext.getImageData(-1, -1, imgBitmap.width + 2, imgBitmap.height + 2);
-        this.offscreenCanvasContext.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
-        return new RGBAImage({width: imgData.width, height: imgData.height}, imgData.data);
-    }
-
     removeTile(params: TileParameters) {
         const loaded = this.loaded,
             uid = params.uid;

diff --git a/src/util/offscreen_canvas_distorted.test.ts b/src/util/offscreen_canvas_distorted.test.ts
@@ -0,0 +1,13 @@
+import {isOffscreenCanvasDistorted} from './offscreen_canvas_distorted';
+import {Canvas} from 'canvas';
+import {offscreenCanvasSupported} from './offscreen_canvas_supported';
+
+test('normal operation does not mangle canvas', () => {
+    const OffscreenCanvas = (window as any).OffscreenCanvas = jest.fn((width:number, height: number) => {
+        return new Canvas(width, height);
+    });
+    expect(offscreenCanvasSupported()).toBeTruthy();
+    OffscreenCanvas.mockClear();
+    expect(isOffscreenCanvasDistorted()).toBeFalsy();
+    expect(OffscreenCanvas).toHaveBeenCalledTimes(1);
+});
diff --git a/src/util/offscreen_canvas_distorted.ts b/src/util/offscreen_canvas_distorted.ts
@@ -0,0 +1,39 @@
+import {offscreenCanvasSupported} from './offscreen_canvas_supported';
+
+let offscreenCanvasDistorted: boolean;
+
+/**
+ * Some browsers don't return the exact pixels from a canvas to prevent user fingerprinting (see #3185).
+ * This function writes pixels to an OffscreenCanvas and reads them back using getImageData, returning false
+ * if they don't match.
+ *
+ * @returns true if the browser supports OffscreenCanvas but it distorts getImageData results, false otherwise.
+ */
+export function isOffscreenCanvasDistorted(): boolean {
+    if (offscreenCanvasDistorted == null) {
+        offscreenCanvasDistorted = false;
+        if (offscreenCanvasSupported()) {
+            const size = 5;
+            const canvas = new OffscreenCanvas(size, size);
+            const context = canvas.getContext('2d', {willReadFrequently: true});
+            if (context) {
+                // fill each pixel with an RGB value that should make the byte at index i equal to i (except alpha channel):
+                // [0, 1, 2, 255, 4, 5, 6, 255, 8, 9, 10, 255, ...]
+                for (let i = 0; i < size * size; i++) {
+                    const base = i * 4;
+                    context.fillStyle = `rgb(${base},${base + 1},${base + 2})`;
+                    context.fillRect(i % size, Math.floor(i / size), 1, 1);
+                }
+                const data = context.getImageData(0, 0, size, size).data;
+                for (let i = 0; i < size * size * 4; i++) {
+                    if (i % 4 !== 3 && data[i] !== i) {
+                        offscreenCanvasDistorted = true;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    return offscreenCanvasDistorted || false;
+}
diff --git a/src/util/util.test.ts b/src/util/util.test.ts
@@ -1,5 +1,6 @@
 import Point from '@mapbox/point-geometry';
-import {arraysIntersect, asyncAll, bezier, clamp, clone, deepEqual, easeCubicInOut, extend, filterObject, findLineIntersection, isClosedPolygon, isCounterClockwise, isPowerOfTwo, keysDifference, mapObject, nextPowerOfTwo, parseCacheControl, pick, uniqueId, wrap} from './util';
+import {arraysIntersect, asyncAll, bezier, clamp, clone, deepEqual, easeCubicInOut, extend, filterObject, findLineIntersection, isClosedPolygon, isCounterClockwise, isPowerOfTwo, keysDifference, mapObject, nextPowerOfTwo, parseCacheControl, pick, readImageDataUsingOffscreenCanvas, readImageUsingVideoFrame, uniqueId, wrap} from './util';
+import {Canvas} from 'canvas';
 
 describe('util', () => {
     expect(easeCubicInOut(0)).toBe(0);
@@ -346,3 +347,172 @@ describe('util findLineIntersection', () => {
         expect(intersection).toBeNull();
     });
 });
+
+describe('util readImageUsingVideoFrame', () => {
+    let format = 'RGBA';
+    const frame = {
+        get format() {
+            return format;
+        },
+        copyTo: jest.fn(buf => {
+            buf.set(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]).subarray(0, buf.length));
+            return Promise.resolve();
+        }),
+        close: jest.fn(),
+    };
+    (window as any).VideoFrame = jest.fn(() => frame);
+    const canvas = document.createElement('canvas');
+    canvas.width = canvas.height = 2;
+
+    beforeEach(() => {
+        format = 'RGBA';
+        frame.copyTo.mockClear();
+        frame.close.mockReset();
+    });
+
+    test('copy RGB', async () => {
+        format = 'RGBA';
+        const result = await readImageUsingVideoFrame(canvas, 0, 0, 2, 2);
+        expect(result).toHaveLength(4 * 4);
+        expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+            layout: [{offset: 0, stride: 8}],
+            rect: {x: 0, y: 0, width: 2, height: 2}
+        });
+        expect(result).toEqual(new Uint8ClampedArray([
+            1, 2, 3, 4, 5, 6, 7, 8,
+            9, 10, 11, 12, 13, 14, 15, 16
+        ]));
+        expect(frame.close).toHaveBeenCalledTimes(1);
+    });
+
+    test('flip BRG', async () => {
+        format = 'BGRX';
+        const result = await readImageUsingVideoFrame(canvas, 0, 0, 2, 2);
+        expect(result).toEqual(new Uint8ClampedArray([
+            3, 2, 1, 4, 7, 6, 5, 8,
+            11, 10, 9, 12, 15, 14, 13, 16
+        ]));
+        expect(frame.close).toHaveBeenCalledTimes(1);
+    });
+
+    test('ignore bad format', async () => {
+        format = 'OTHER';
+        await expect(readImageUsingVideoFrame(canvas, 0, 0, 2, 2)).rejects.toThrow();
+        expect(frame.close).toHaveBeenCalledTimes(1);
+    });
+
+    describe('layout/rect', () => {
+        beforeEach(() => {
+            (window as any).VideoFrame = jest.fn(() => frame);
+            canvas.width = canvas.height = 3;
+        });
+
+        test('full rectangle', async () => {
+            await readImageUsingVideoFrame(canvas, 0, 0, 3, 3);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 12}],
+                rect: {x: 0, y: 0, width: 3, height: 3}
+            });
+        });
+
+        test('top left', async () => {
+            await readImageUsingVideoFrame(canvas, 0, 0, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 8}],
+                rect: {x: 0, y: 0, width: 2, height: 2}
+            });
+        });
+
+        test('top right', async () => {
+            await readImageUsingVideoFrame(canvas, 1, 0, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 8}],
+                rect: {x: 1, y: 0, width: 2, height: 2}
+            });
+        });
+
+        test('bottom left', async () => {
+            await readImageUsingVideoFrame(canvas, 0, 1, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 8}],
+                rect: {x: 0, y: 1, width: 2, height: 2}
+            });
+        });
+
+        test('bottom right', async () => {
+            await readImageUsingVideoFrame(canvas, 1, 1, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 8}],
+                rect: {x: 1, y: 1, width: 2, height: 2}
+            });
+        });
+
+        test('middle', async () => {
+            await readImageUsingVideoFrame(canvas, 1, 1, 1, 1);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 4}],
+                rect: {x: 1, y: 1, width: 1, height: 1}
+            });
+        });
+
+        test('extend past on all sides', async () => {
+            await readImageUsingVideoFrame(canvas, -1, -1, 5, 5);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 4 * 5 + 4, stride: 4 * 5}],
+                rect: {x: 0, y: 0, width: 3, height: 3}
+            });
+        });
+
+        test('overhang top left', async () => {
+            await readImageUsingVideoFrame(canvas, -1, -1, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 4 * 2 + 4, stride: 4 * 2}],
+                rect: {x: 0, y: 0, width: 1, height: 1}
+            });
+        });
+
+        test('overhang top right', async () => {
+            await readImageUsingVideoFrame(canvas, 2, -1, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 4 * 2, stride: 4 * 2}],
+                rect: {x: 2, y: 0, width: 1, height: 1}
+            });
+        });
+
+        test('overhang bottom left', async () => {
+            await readImageUsingVideoFrame(canvas, -1, 2, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 4, stride: 4 * 2}],
+                rect: {x: 0, y: 2, width: 1, height: 1}
+            });
+        });
+
+        test('overhang bottom right', async () => {
+            await readImageUsingVideoFrame(canvas, 2, 2, 2, 2);
+            expect(frame.copyTo).toHaveBeenCalledWith(expect.anything(), {
+                layout: [{offset: 0, stride: 4 * 2}],
+                rect: {x: 2, y: 2, width: 1, height: 1}
+            });
+        });
+    });
+});
+
+describe('util readImageDataUsingOffscreenCanvas', () => {
+    test('reads pixels from image', async () => {
+        (window as any).OffscreenCanvas = Canvas;
+        const image = new Canvas(2, 2);
+        const context = image.getContext('2d');
+        context.fillStyle = 'rgb(10,0,0)';
+        context.fillRect(0, 0, 1, 1);
+        context.fillStyle = 'rgb(0,20,0)';
+        context.fillRect(1, 0, 1, 1);
+        context.fillStyle = 'rgb(0,0,30)';
+        context.fillRect(0, 1, 1, 1);
+        context.fillStyle = 'rgb(40,40,40)';
+        context.fillRect(1, 1, 1, 1);
+        expect([...await readImageDataUsingOffscreenCanvas(image as any, 0, 0, 2, 2)]).toEqual([
+            10, 0, 0, 255, 0, 20, 0, 255,
+            0, 0, 30, 255, 40, 40, 40, 255,
+        ]);
+    });
+});