From 174b5a5d55ddeaac1f50bf88870b2027d362af2c Mon Sep 17 00:00:00 2001 From: wang1212 Date: Wed, 23 Oct 2024 11:31:44 +0800 Subject: [PATCH] perf: optimize large image rendering performance --- .changeset/nine-mangos-shop.md | 9 + __tests__/demos/perf/image.ts | 59 +++++ __tests__/demos/perf/index.ts | 1 + __tests__/main.ts | 1 + packages/g-lite/src/Canvas.ts | 2 + .../src/services/OffscreenCanvasCreator.ts | 19 ++ packages/g-lite/src/types.ts | 20 ++ .../src/shapes/styles/Default.ts | 1 + .../src/shapes/styles/Image.ts | 227 +++++++++++++++-- .../src/utils/math.ts | 55 ++++ .../src/CanvaskitRendererPlugin.ts | 17 +- .../src/renderers/Image.ts | 20 +- .../src/TexturePool.ts | 8 +- .../g-plugin-image-loader/src/ImagePool.ts | 236 ++++++++++++++++-- .../g-plugin-image-loader/src/ImageSlicer.ts | 135 ++++++++++ .../src/LoadImagePlugin.ts | 58 +++-- .../src/RefCountCache.ts | 75 ++++++ packages/g-plugin-image-loader/src/index.ts | 10 +- rollup.config.mjs | 4 +- 19 files changed, 870 insertions(+), 87 deletions(-) create mode 100644 .changeset/nine-mangos-shop.md create mode 100644 __tests__/demos/perf/image.ts create mode 100644 packages/g-plugin-image-loader/src/ImageSlicer.ts create mode 100644 packages/g-plugin-image-loader/src/RefCountCache.ts diff --git a/.changeset/nine-mangos-shop.md b/.changeset/nine-mangos-shop.md new file mode 100644 index 000000000..320d77007 --- /dev/null +++ b/.changeset/nine-mangos-shop.md @@ -0,0 +1,9 @@ +--- +'@antv/g-plugin-canvaskit-renderer': patch +'@antv/g-plugin-canvas-renderer': patch +'@antv/g-plugin-device-renderer': patch +'@antv/g-plugin-image-loader': patch +'@antv/g-lite': patch +--- + +perf: optimize large image rendering performance diff --git a/__tests__/demos/perf/image.ts b/__tests__/demos/perf/image.ts new file mode 100644 index 000000000..e6239a755 --- /dev/null +++ b/__tests__/demos/perf/image.ts @@ -0,0 +1,59 @@ +import { Canvas, Image as GImage } from '@antv/g'; + +export async function image(context: { canvas: Canvas }) { + const { canvas } = context; + await canvas.ready; + console.log(canvas); + + const $dom = canvas.getContextService().getDomElement() as HTMLCanvasElement; + $dom.style.border = '1px solid gray'; + + // --- + $dom.addEventListener('wheel', (event) => { + event.preventDefault(); + + const { deltaX, deltaY } = event; + const d = -(deltaX ?? deltaY); + + const ratio = 1 + (Math.min(Math.max(d, -50), 50) * 1) / 100; + const zoom = canvas.getCamera().getZoom(); + + canvas + .getCamera() + .setZoomByViewportPoint(zoom * ratio, [event.offsetX, event.offsetY]); + }); + + let isDragging = false; + let lastX, lastY; + $dom.addEventListener('mousedown', (e) => { + isDragging = true; + lastX = e.clientX; + lastY = e.clientY; + }); + $dom.addEventListener('mousemove', (e) => { + if (isDragging) { + const dx = e.clientX - lastX; + const dy = e.clientY - lastY; + canvas.getCamera().pan(-dx, -dy); + lastX = e.clientX; + lastY = e.clientY; + } + }); + $dom.addEventListener('mouseup', () => { + isDragging = false; + }); + + // --- + + let image = new GImage({ + style: { + x: 0, + y: 0, + // width: 100, + // height: 400, + // src: 'https://gw.alipayobjects.com/mdn/rms_6ae20b/afts/img/A*N4ZMS7gHsUIAAAAAAAAAAABkARQnAQ', + src: 'http://mmtcdp.stable.alipay.net/cto_designhubcore/afts/img/g1a5QYkvbcMAAAAAAAAAAAAADgLVAQBr/original', + }, + }); + canvas.appendChild(image); +} diff --git a/__tests__/demos/perf/index.ts b/__tests__/demos/perf/index.ts index 15475060b..7534392e1 100644 --- a/__tests__/demos/perf/index.ts +++ b/__tests__/demos/perf/index.ts @@ -1,2 +1,3 @@ export { circles } from './circles'; export { rects } from './rect'; +export { image } from './image'; diff --git a/__tests__/main.ts b/__tests__/main.ts index e077f11f1..94ca49aaa 100644 --- a/__tests__/main.ts +++ b/__tests__/main.ts @@ -1,4 +1,5 @@ import * as lil from 'lil-gui'; +import '@antv/g-camera-api'; import { Canvas, CanvasEvent, runtime } from '@antv/g'; import { Renderer as CanvasRenderer } from '@antv/g-canvas'; import { Renderer as CanvaskitRenderer } from '@antv/g-canvaskit'; diff --git a/packages/g-lite/src/Canvas.ts b/packages/g-lite/src/Canvas.ts index 0ad0a3bad..05b880aeb 100644 --- a/packages/g-lite/src/Canvas.ts +++ b/packages/g-lite/src/Canvas.ts @@ -167,6 +167,7 @@ export class Canvas extends EventTarget implements ICanvas { requestAnimationFrame, cancelAnimationFrame, createImage, + enableLargeImageOptimization, supportsPointerEvents, supportsTouchEvents, supportsCSSTransform, @@ -244,6 +245,7 @@ export class Canvas extends EventTarget implements ICanvas { cursor: cursor || ('default' as Cursor), background: background || 'transparent', createImage, + enableLargeImageOptimization, document, supportsCSSTransform, useNativeClickEvent, diff --git a/packages/g-lite/src/services/OffscreenCanvasCreator.ts b/packages/g-lite/src/services/OffscreenCanvasCreator.ts index 112c3414f..ebd2939ad 100644 --- a/packages/g-lite/src/services/OffscreenCanvasCreator.ts +++ b/packages/g-lite/src/services/OffscreenCanvasCreator.ts @@ -13,6 +13,25 @@ export class OffscreenCanvasCreator { private canvas: CanvasLike; private context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + /** + * @returns new canvas instance + */ + static createCanvas(): HTMLCanvasElement | OffscreenCanvas | null { + try { + return new window.OffscreenCanvas(0, 0); + } catch { + // + } + + try { + return document.createElement('canvas'); + } catch { + // + } + + return null; + } + getOrCreateCanvas( offscreenCanvas: CanvasLike, contextAttributes?: CanvasRenderingContext2DSettings, diff --git a/packages/g-lite/src/types.ts b/packages/g-lite/src/types.ts index 41108e2d2..f323e51e5 100644 --- a/packages/g-lite/src/types.ts +++ b/packages/g-lite/src/types.ts @@ -516,6 +516,26 @@ export interface CanvasConfig { * replace `new window.Image()` */ createImage?: (src: string) => HTMLImageElement; + /** + * Optimize rendering performance for high-resolution large images + */ + enableLargeImageOptimization?: + | boolean + | { + /** + * Downsampling rate threshold, [0.01, 0.5], represents the ratio of reducing the image. + * For example, 0.3 means reducing the original image to 0.3 times of the original. + * + * default 0.5 + */ + downSamplingRateThreshold?: number; + /** + * The maximum size of the downsampled image, with equal width and height. + * + * default 2048 + */ + maxDownSampledImageSize?: number; + }; /** * limits query diff --git a/packages/g-plugin-canvas-renderer/src/shapes/styles/Default.ts b/packages/g-plugin-canvas-renderer/src/shapes/styles/Default.ts index a1b1535e0..babd1c1e6 100644 --- a/packages/g-plugin-canvas-renderer/src/shapes/styles/Default.ts +++ b/packages/g-plugin-canvas-renderer/src/shapes/styles/Default.ts @@ -225,6 +225,7 @@ export function getPattern( } const canvasPattern = imagePool.getOrCreatePatternSync( + object, pattern, context, $offscreenCanvas, diff --git a/packages/g-plugin-canvas-renderer/src/shapes/styles/Image.ts b/packages/g-plugin-canvas-renderer/src/shapes/styles/Image.ts index 8a217673b..b7a43cfa8 100644 --- a/packages/g-plugin-canvas-renderer/src/shapes/styles/Image.ts +++ b/packages/g-plugin-canvas-renderer/src/shapes/styles/Image.ts @@ -1,12 +1,143 @@ import type { DisplayObject, ParsedImageStyleProps } from '@antv/g-lite'; -import type { ImagePool } from '@antv/g-plugin-image-loader'; -import { isNil, isString } from '@antv/util'; +import { ImagePool, type ImageCache } from '@antv/g-plugin-image-loader'; +import { isNil } from '@antv/util'; +import { mat4 } from 'gl-matrix'; import { setShadowAndFilter } from './Default'; import type { StyleRenderer } from './interfaces'; +import { transformRect, calculateOverlapRect } from '../../utils/math'; export class ImageRenderer implements StyleRenderer { constructor(private imagePool: ImagePool) {} + static renderFull( + context: CanvasRenderingContext2D, + parsedStyle: ParsedImageStyleProps, + object: DisplayObject, + data: { + image: HTMLImageElement; + drawRect: [number, number, number, number]; + }, + ) { + context.drawImage( + data.image, + Math.floor(data.drawRect[0]), + Math.floor(data.drawRect[1]), + Math.ceil(data.drawRect[2]), + Math.ceil(data.drawRect[3]), + ); + } + + #renderDownSampled( + context: CanvasRenderingContext2D, + parsedStyle: ParsedImageStyleProps, + object: DisplayObject, + data: { + src: string | HTMLImageElement; + imageCache: ImageCache; + drawRect: [number, number, number, number]; + }, + ) { + const { src, imageCache } = data; + + if (!imageCache.downSampled) { + this.imagePool + .createDownSampledImage(src, object) + .then((res) => { + // rerender + object.renderable.dirty = true; + object.ownerDocument.defaultView.context.renderingService.dirtify(); + }) + .catch(() => { + // + }); + + return; + } + + context.drawImage( + imageCache.downSampled, + Math.floor(data.drawRect[0]), + Math.floor(data.drawRect[1]), + Math.ceil(data.drawRect[2]), + Math.ceil(data.drawRect[3]), + ); + } + + #renderTile( + context: CanvasRenderingContext2D, + parsedStyle: ParsedImageStyleProps, + object: DisplayObject, + data: { + src: string | HTMLImageElement; + imageCache: ImageCache; + imageRect: [number, number, number, number]; + drawRect: [number, number, number, number]; + }, + ) { + const { src, imageCache, imageRect, drawRect } = data; + const { size: originalSize } = imageCache; + const { a, b, c, d, e, f } = context.getTransform(); + + context.resetTransform(); + + if (!imageCache?.gridSize) { + this.imagePool + .createImageTiles(src, [], object) + .then(() => { + // rerender + object.renderable.dirty = true; + object.ownerDocument.defaultView.context.renderingService.dirtify(); + }) + .catch(() => { + // + }); + + return; + } + + const scaleToOrigin = [ + originalSize[0] / imageRect[2], + originalSize[1] / imageRect[3], + ]; + const scaledTileSize = [ + imageCache.tileSize[0] / scaleToOrigin[0], + imageCache.tileSize[1] / scaleToOrigin[1], + ]; + const [startTileX, endTileX] = [ + Math.floor((drawRect[0] - imageRect[0]) / scaledTileSize[0]), + Math.ceil((drawRect[0] + drawRect[2] - imageRect[0]) / scaledTileSize[0]), + ]; + const [startTileY, endTileY] = [ + Math.floor((drawRect[1] - imageRect[1]) / scaledTileSize[1]), + Math.ceil((drawRect[1] + drawRect[3] - imageRect[1]) / scaledTileSize[1]), + ]; + + for (let tileY = startTileY; tileY <= endTileY; tileY++) { + for (let tileX = startTileX; tileX <= endTileX; tileX++) { + const item = imageCache.tiles[tileY][tileX]; + + if (item) { + const tileRect = [ + Math.floor(imageRect[0] + item.tileX * scaledTileSize[0]), + Math.floor(imageRect[1] + item.tileY * scaledTileSize[1]), + Math.ceil(scaledTileSize[0]), + Math.ceil(scaledTileSize[1]), + ]; + + context.drawImage( + item.data, + tileRect[0], + tileRect[1], + tileRect[2], + tileRect[3], + ); + } + } + } + + context.setTransform(a, b, c, d, e, f); + } + render( context: CanvasRenderingContext2D, parsedStyle: ParsedImageStyleProps, @@ -22,28 +153,86 @@ export class ImageRenderer implements StyleRenderer { shadowBlur, } = parsedStyle; - let image: HTMLImageElement; + const imageCache = this.imagePool.getImageSync(src, object); + const image = imageCache?.img; let iw = width; let ih = height; - if (isString(src)) { - // image has been loaded in `mounted` hook - image = this.imagePool.getImageSync(src); - } else { - iw ||= src.width; - ih ||= src.height; - image = src; + if (!image) { + return; } - if (image) { - const hasShadow = !isNil(shadowColor) && shadowBlur > 0; - setShadowAndFilter(object, context, hasShadow); + iw ||= image.width; + ih ||= image.height; - // node-canvas will throw the following err: - // Error: Image given has not completed loading - try { - context.drawImage(image, x, y, iw, ih); - } catch {} - } + const hasShadow = !isNil(shadowColor) && shadowBlur > 0; + setShadowAndFilter(object, context, hasShadow); + + // node-canvas will throw the following err: + // Error: Image given has not completed loading + try { + const { width: viewWidth, height: viewHeight } = + object.ownerDocument.defaultView.getContextService().getDomElement(); + + const currentTransform = context.getTransform(); + const { a, b, c, d, e, f } = currentTransform; + // 构建 mat4 矩阵 + // prettier-ignore + const transformMatrix = mat4.fromValues( + a, c, 0, 0, + b, d, 0, 0, + 0, 0, 1, 0, + e, f, 0, 1, + ); + const imageRect = transformRect([x, y, iw, ih], transformMatrix); + const drawRect = calculateOverlapRect( + [0, 0, viewWidth, viewHeight], + imageRect, + ); + + if (!drawRect) { + return; + } + + if ( + !object.ownerDocument.defaultView.getConfig() + .enableLargeImageOptimization + ) { + ImageRenderer.renderFull(context, parsedStyle, object, { + image, + drawRect: [x, y, iw, ih], + }); + + return; + } + + const sizeOfOrigin = imageRect[2] / imageCache.size[0]; + + if (sizeOfOrigin < (imageCache.downSamplingRate || 0.5)) { + this.#renderDownSampled(context, parsedStyle, object, { + src, + imageCache, + drawRect: [x, y, iw, ih], + }); + + return; + } + + if (!ImagePool.isSupportTile) { + ImageRenderer.renderFull(context, parsedStyle, object, { + image, + drawRect: [x, y, iw, ih], + }); + + return; + } + + this.#renderTile(context, parsedStyle, object, { + src, + imageCache, + imageRect, + drawRect, + }); + } catch {} } } diff --git a/packages/g-plugin-canvas-renderer/src/utils/math.ts b/packages/g-plugin-canvas-renderer/src/utils/math.ts index ee9353f47..f21d86b3b 100644 --- a/packages/g-plugin-canvas-renderer/src/utils/math.ts +++ b/packages/g-plugin-canvas-renderer/src/utils/math.ts @@ -1,3 +1,5 @@ +import { vec3, mat4 } from 'gl-matrix'; + /** * 判断两个点是否重合，点坐标的格式为 [x, y] */ @@ -7,3 +9,56 @@ export function isSamePoint( ) { return point1[0] === point2[0] && point1[1] === point2[1]; } + +export function calculateOverlapRect< + Rect extends [number, number, number, number], +>(rect1: Rect, rect2: Rect): null | Rect { + const [x1, y1, w1, h1] = rect1; + const [x2, y2, w2, h2] = rect2; + + // 计算重叠区域的左上角和右下角 + const overlapLeft = Math.max(x1, x2); + const overlapTop = Math.max(y1, y2); + const overlapRight = Math.min(x1 + w1, x2 + w2); + const overlapBottom = Math.min(y1 + h1, y2 + h2); + + if (overlapRight <= overlapLeft || overlapBottom <= overlapTop) { + return null; + } + + return [ + overlapLeft, + overlapTop, + overlapRight - overlapLeft, + overlapBottom - overlapTop, + ] as Rect; +} + +export function transformRect( + rect: Rect, + matrix: mat4, +): Rect { + const tl = vec3.transformMat4(vec3.create(), [rect[0], rect[1], 0], matrix); + const tr = vec3.transformMat4( + vec3.create(), + [rect[0] + rect[2], rect[1], 0], + matrix, + ); + const bl = vec3.transformMat4( + vec3.create(), + [rect[0], rect[1] + rect[3], 0], + matrix, + ); + const br = vec3.transformMat4( + vec3.create(), + [rect[0] + rect[2], rect[1] + rect[3], 0], + matrix, + ); + + return [ + Math.min(tl[0], tr[0], bl[0], br[0]), + Math.min(tl[1], tr[1], bl[1], br[1]), + Math.max(tl[0], tr[0], bl[0], br[0]) - Math.min(tl[0], tr[0], bl[0], br[0]), + Math.max(tl[1], tr[1], bl[1], br[1]) - Math.min(tl[1], tr[1], bl[1], br[1]), + ] as Rect; +} diff --git a/packages/g-plugin-canvaskit-renderer/src/CanvaskitRendererPlugin.ts b/packages/g-plugin-canvaskit-renderer/src/CanvaskitRendererPlugin.ts index c9df00259..3b279c9aa 100644 --- a/packages/g-plugin-canvaskit-renderer/src/CanvaskitRendererPlugin.ts +++ b/packages/g-plugin-canvaskit-renderer/src/CanvaskitRendererPlugin.ts @@ -320,12 +320,17 @@ export class CanvaskitRendererPlugin implements RenderingPlugin { let src: TextureSource; if (isString(image)) { - // @ts-ignore - src = (this.context.imagePool as ImagePool).getImageSync(image, () => { - // set dirty rectangle flag - object.renderable.dirty = true; - this.context.renderingService.dirtify(); - }); + const imageCache = (this.context.imagePool as ImagePool).getImageSync( + image, + object, + () => { + // set dirty rectangle flag + object.renderable.dirty = true; + this.context.renderingService.dirtify(); + }, + ); + + src = imageCache?.img; } else if ((image as Rect).nodeName === 'rect') { // image.forEach((object: DisplayObject) => { // }); diff --git a/packages/g-plugin-canvaskit-renderer/src/renderers/Image.ts b/packages/g-plugin-canvaskit-renderer/src/renderers/Image.ts index 882b1d1d2..e64ef9330 100644 --- a/packages/g-plugin-canvaskit-renderer/src/renderers/Image.ts +++ b/packages/g-plugin-canvaskit-renderer/src/renderers/Image.ts @@ -5,7 +5,6 @@ import type { ContextService, } from '@antv/g-lite'; import type { ImagePool } from '@antv/g-plugin-image-loader'; -import { isString } from '@antv/util'; import type { CanvasKitContext, RendererContribution, @@ -29,21 +28,18 @@ export class ImageRenderer implements RendererContribution { const { x, y, width, height, src, fillOpacity, opacity } = object.parsedStyle as ParsedImageStyleProps; - let image: HTMLImageElement; + const imageCache = (this.context.imagePool as ImagePool).getImageSync( + src, + object, + ); + const image = imageCache?.img; let iw = width; let ih = height; - if (isString(src)) { - // image has been loaded in `mounted` hook - // @ts-ignore - image = (this.context.imagePool as ImagePool).getImageSync(src); - } else { - iw ||= src.width; - ih ||= src.height; - image = src; - } - if (image) { + iw ||= image.width; + ih ||= image.height; + const decoded = surface.makeImageFromTextureSource( image, // { diff --git a/packages/g-plugin-device-renderer/src/TexturePool.ts b/packages/g-plugin-device-renderer/src/TexturePool.ts index 6cbe7e8ea..b09ff1b4a 100644 --- a/packages/g-plugin-device-renderer/src/TexturePool.ts +++ b/packages/g-plugin-device-renderer/src/TexturePool.ts @@ -189,8 +189,12 @@ export class TexturePool { let src: CanvasImageSource; // Image URL if (isString(image)) { - // @ts-ignore - src = this.context.imagePool.getImageSync(image, callback); + const imageCache = (this.context.imagePool as ImagePool).getImageSync( + image, + instance, + callback, + ); + src = imageCache?.img; } else { src = image as CanvasImageSource; } diff --git a/packages/g-plugin-image-loader/src/ImagePool.ts b/packages/g-plugin-image-loader/src/ImagePool.ts index e14afb564..bda1ff93f 100644 --- a/packages/g-plugin-image-loader/src/ImagePool.ts +++ b/packages/g-plugin-image-loader/src/ImagePool.ts @@ -1,5 +1,4 @@ import { - CanvasConfig, DisplayObject, GradientType, LinearGradient, @@ -13,9 +12,25 @@ import { isBrowser, parseTransform, parsedTransformToMat4, + Image, + OffscreenCanvasCreator, + type CanvasContext, + type GlobalRuntime, } from '@antv/g-lite'; import { isString } from '@antv/util'; import { mat4 } from 'gl-matrix'; +import { RefCountCache } from './RefCountCache'; +import { type SliceResult, ImageSlicer } from './ImageSlicer'; + +export interface ImageCache extends Partial { + img: HTMLImageElement; + /** [width, height] */ + size: [number, number]; + downSampled?: ImageBitmap | HTMLImageElement; + downSamplingRate?: number; +} + +const IMAGE_CACHE = new RefCountCache(); export type GradientParams = (LinearGradient & RadialGradient) & { width: number; @@ -26,58 +41,212 @@ export type GradientParams = (LinearGradient & RadialGradient) & { min: [number, number]; type: GradientType; }; + export class ImagePool { + static isSupportTile = !!OffscreenCanvasCreator.createCanvas(); private imageCache: Record = {}; private gradientCache: Record = {}; private patternCache: Record = {}; - constructor(private canvasConfig: Partial) {} + constructor( + public context: CanvasContext, + private runtime: GlobalRuntime, + ) {} - getImageSync(src: string, callback?: (img: HTMLImageElement) => void) { - if (!this.imageCache[src]) { - this.getOrCreateImage(src).then((img) => { - if (callback) { - callback(img); - } - }); - } else if (callback) { - callback(this.imageCache[src]); + getImageSync( + src: Image['attributes']['src'], + ref: DisplayObject, + callback?: (cache: ImageCache) => void, + ): ImageCache | null { + const imageSource = isString(src) ? src : src.src; + + if (IMAGE_CACHE.has(imageSource)) { + const imageCache = IMAGE_CACHE.get(imageSource, ref); + + if (imageCache.img.complete) { + callback?.(imageCache); + + return imageCache; + } } - return this.imageCache[src]; + this.getOrCreateImage(src, ref) + .then((cache) => { + callback?.(cache); + }) + .catch(() => { + // + }); + + return null; } - getOrCreateImage(src: string): Promise { - if (this.imageCache[src]) { - return Promise.resolve(this.imageCache[src]); + getOrCreateImage( + src: Image['attributes']['src'], + ref: DisplayObject, + ): Promise { + const imageSource = isString(src) ? src : src.src; + + if (!isString(src) && !IMAGE_CACHE.has(imageSource)) { + const imageCache: ImageCache = { + img: src, + size: [src.naturalWidth || src.width, src.naturalHeight || src.height], + tileSize: calculateImageTileSize(src), + }; + + IMAGE_CACHE.put(imageSource, imageCache, ref); + } + + if (IMAGE_CACHE.has(imageSource)) { + const imageCache = IMAGE_CACHE.get(imageSource, ref); + + if (imageCache.img.complete) { + return Promise.resolve(imageCache); + } + + return new Promise((resolve, reject) => { + imageCache.img.addEventListener('load', () => { + imageCache.tileSize = calculateImageTileSize(imageCache.img); + resolve(imageCache); + }); + + imageCache.img.addEventListener('error', (ev) => { + reject(ev); + }); + }); } // @see https://github.com/antvis/g/issues/938 - const { createImage } = this.canvasConfig; + const { createImage } = this.context.config; return new Promise((resolve, reject) => { let image: HTMLImageElement; if (createImage) { - image = createImage(src); + image = createImage(imageSource); } else if (isBrowser) { image = new window.Image(); } if (image) { + const imageCache: ImageCache = { + img: image, + size: [0, 0], + tileSize: calculateImageTileSize(image), + }; + + IMAGE_CACHE.put(imageSource, imageCache, ref); + image.onload = () => { - this.imageCache[src] = image; - resolve(image); + imageCache.size = [ + image.naturalWidth || image.width, + image.naturalHeight || image.height, + ]; + imageCache.tileSize = calculateImageTileSize(imageCache.img); + resolve(imageCache); }; image.onerror = (ev) => { reject(ev); }; image.crossOrigin = 'Anonymous'; - image.src = src; + image.src = imageSource; } }); } + async createDownSampledImage( + src: Image['attributes']['src'], + ref: DisplayObject, + ): Promise { + const imageCache = await this.getOrCreateImage(src, ref); + if (typeof imageCache.downSamplingRate !== 'undefined') { + return imageCache; + } + + const { enableLargeImageOptimization } = this.context.config; + const { maxDownSampledImageSize = 2048, downSamplingRateThreshold = 0.5 } = + typeof enableLargeImageOptimization === 'boolean' + ? {} + : enableLargeImageOptimization; + const createImageBitmapFunc = this.runtime.globalThis + .createImageBitmap as typeof createImageBitmap; + const [originWidth, originHeight] = imageCache.size; + let resizedImage: ImageCache['downSampled'] = imageCache.img; + let downSamplingRate = Math.min( + (maxDownSampledImageSize + maxDownSampledImageSize) / + (originWidth + originHeight), + Math.max(0.01, Math.min(downSamplingRateThreshold, 0.5)), + ); + + let updateCache: ImageCache = { + ...imageCache, + downSamplingRate, + }; + + IMAGE_CACHE.update(imageCache.img.src, updateCache, ref); + + if (createImageBitmapFunc) { + try { + resizedImage = await createImageBitmapFunc(imageCache.img, { + resizeWidth: originWidth * downSamplingRate, + resizeHeight: originHeight * downSamplingRate, + }); + } catch { + downSamplingRate = 1; + } + } else { + downSamplingRate = 1; + } + + updateCache = { + ...this.getImageSync(src, ref), + downSampled: resizedImage, + downSamplingRate, + }; + + IMAGE_CACHE.update(imageCache.img.src, updateCache, ref); + + return updateCache; + } + + async createImageTiles( + src: Image['attributes']['src'], + tiles: [number, number][], + ref: DisplayObject, + ): Promise { + const imageCache = await this.getOrCreateImage(src, ref); + const { requestAnimationFrame, cancelAnimationFrame } = + ref.ownerDocument.defaultView; + + ImageSlicer.api = { + requestAnimationFrame, + cancelAnimationFrame, + createCanvas: () => OffscreenCanvasCreator.createCanvas(), + }; + + const updateCache: ImageCache = { + ...imageCache, + ...ImageSlicer.sliceImage( + imageCache.img, + imageCache.tileSize[0], + imageCache.tileSize[0], + ), + }; + + IMAGE_CACHE.update(imageCache.img.src, updateCache, ref); + + return updateCache; + } + + releaseImage(src: Image['attributes']['src'], ref: DisplayObject) { + IMAGE_CACHE.release(isString(src) ? src : src.src, ref); + } + + releaseImageRef(ref: DisplayObject) { + IMAGE_CACHE.releaseRef(ref); + } + getOrCreatePatternSync( + object: DisplayObject, pattern: Pattern, context: CanvasRenderingContext2D, $offscreenCanvas: HTMLCanvasElement, @@ -95,7 +264,8 @@ export class ImagePool { let needScaleWithDPR = false; // Image URL if (isString(image)) { - src = this.getImageSync(image, callback); + const imageCache = this.getImageSync(image, object, callback); + src = imageCache?.img; } else if ($offscreenCanvas) { src = $offscreenCanvas; needScaleWithDPR = true; @@ -206,3 +376,27 @@ export class ImagePool { } } } + +function calculateImageTileSize(img: HTMLImageElement): [number, number] { + if (!img.complete) { + return [0, 0]; + } + + const [width, height] = [ + img.naturalWidth || img.width, + img.naturalHeight || img.height, + ]; + + let tileSize = 256; + + [256, 512].forEach((size) => { + const rows = Math.ceil(height / size); + const cols = Math.ceil(width / size); + + if (rows * cols < 1e3) { + tileSize = size; + } + }); + + return [tileSize, tileSize]; +} diff --git a/packages/g-plugin-image-loader/src/ImageSlicer.ts b/packages/g-plugin-image-loader/src/ImageSlicer.ts new file mode 100644 index 000000000..cf1e1e1ca --- /dev/null +++ b/packages/g-plugin-image-loader/src/ImageSlicer.ts @@ -0,0 +1,135 @@ +const tasks: (() => void)[] = []; +let nextFrameTasks: (() => void)[] = []; + +interface API { + requestAnimationFrame: typeof requestAnimationFrame; + cancelAnimationFrame: typeof cancelAnimationFrame; + createCanvas: () => HTMLCanvasElement | OffscreenCanvas; +} + +export interface SliceResult { + tileSize: [number, number]; + /** [rows, cols] */ + gridSize: [number, number]; + /** + * @example + * ``` + * [ + * // tileY=0 + * [tileX=0, tileX=1, ...], + * // tileY=1 + * [tileX=0, tileX=1, ...], + * ] + * ``` + */ + tiles: (null | { + x: number; + y: number; + tileX: number; + tileY: number; + data: HTMLCanvasElement | OffscreenCanvas; + })[][]; +} + +export class ImageSlicer { + static api: API; + static TASK_NUM_PER_FRAME = 10; + static rafId: ReturnType; + + static stop(api = ImageSlicer.api) { + if (ImageSlicer.rafId) { + api.cancelAnimationFrame(ImageSlicer.rafId); + ImageSlicer.rafId = null; + } + } + + static executeTask(api = ImageSlicer.api) { + if (tasks.length <= 0 && nextFrameTasks.length <= 0) { + return; + } + + nextFrameTasks.forEach((task) => task()); + nextFrameTasks = tasks.splice(0, ImageSlicer.TASK_NUM_PER_FRAME); + + ImageSlicer.rafId = api.requestAnimationFrame(() => { + ImageSlicer.executeTask(api); + }); + } + + static sliceImage( + image: HTMLImageElement, + sliceWidth: number, + sliceHeight: number, + overlap = 0, + api = ImageSlicer.api, + ) { + const imageWidth = image.naturalWidth || image.width; + const imageHeight = image.naturalHeight || image.height; + + // 计算步长(考虑重叠区域) + const strideW = sliceWidth - overlap; + const strideH = sliceHeight - overlap; + + // 计算网格尺寸 + const gridCols = Math.ceil(imageWidth / strideW); + const gridRows = Math.ceil(imageHeight / strideH); + + const result: SliceResult = { + tileSize: [sliceWidth, sliceHeight], + gridSize: [gridRows, gridCols], + tiles: Array(gridRows) + .fill(null) + .map(() => Array(gridCols).fill(null) as SliceResult['tiles'][number]), + }; + + // 遍历网格创建切片 + for (let row = 0; row < gridRows; row++) { + for (let col = 0; col < gridCols; col++) { + tasks.push(() => { + // 计算当前切片的坐标 + const startX = col * strideW; + const startY = row * strideH; + + // 处理最后一列/行的特殊情况 + const [tempSliceWidth, tempSliceHeight] = [ + Math.min(sliceWidth, imageWidth - startX), + Math.min(sliceHeight, imageHeight - startY), + ]; + + // 创建切片canvas + const sliceCanvas = api.createCanvas(); + sliceCanvas.width = sliceWidth; + sliceCanvas.height = sliceHeight; + const sliceCtx = sliceCanvas.getContext('2d'); + + // 将图像部分绘制到切片canvas上 + sliceCtx.drawImage( + image, + startX, + startY, + tempSliceWidth, + tempSliceHeight, + 0, + 0, + tempSliceWidth, + tempSliceHeight, + ); + + // 存储切片信息 + result.tiles[row][col] = { + x: startX, + y: startY, + tileX: col, + tileY: row, + data: sliceCanvas, + }; + }); + } + } + + ImageSlicer.stop(); + ImageSlicer.executeTask(); + + return result; + } +} diff --git a/packages/g-plugin-image-loader/src/LoadImagePlugin.ts b/packages/g-plugin-image-loader/src/LoadImagePlugin.ts index 46119d649..f332a3f95 100644 --- a/packages/g-plugin-image-loader/src/LoadImagePlugin.ts +++ b/packages/g-plugin-image-loader/src/LoadImagePlugin.ts @@ -1,5 +1,4 @@ import type { - DisplayObject, FederatedEvent, Image, MutationEvent, @@ -37,7 +36,7 @@ export class LoadImagePlugin implements RenderingPlugin { const { src, keepAspectRatio } = attributes; if (isString(src)) { - imagePool.getImageSync(src, ({ width, height }) => { + imagePool.getImageSync(src, object, ({ img: { width, height } }) => { if (keepAspectRatio) { calculateWithAspectRatio(object, width, height); } @@ -51,32 +50,52 @@ export class LoadImagePlugin implements RenderingPlugin { }; const handleAttributeChanged = (e: MutationEvent) => { - const object = e.target as DisplayObject; - const { attrName, newValue } = e; - - if (object.nodeName === Shape.IMAGE) { - if (attrName === 'src') { - if (isString(newValue)) { - imagePool.getOrCreateImage(newValue).then(({ width, height }) => { - if (object.attributes.keepAspectRatio) { - calculateWithAspectRatio(object, width, height); - } - - // set dirty rectangle flag - object.renderable.dirty = true; - renderingService.dirtify(); - }); - } - } + const object = e.target as Image; + const { attrName, prevValue, newValue } = e; + + if (object.nodeName !== Shape.IMAGE || attrName !== 'src') { + return; + } + + if (prevValue !== newValue) { + imagePool.releaseImage(prevValue as Image['attributes']['src'], object); + } + + if (isString(newValue)) { + imagePool + .getOrCreateImage(newValue, object) + .then(({ img: { width, height } }) => { + if (object.attributes.keepAspectRatio) { + calculateWithAspectRatio(object, width, height); + } + + // set dirty rectangle flag + object.renderable.dirty = true; + renderingService.dirtify(); + }) + .catch(() => { + // + }); } }; + function handleDestroy(e: FederatedEvent) { + const object = e.target as Image; + + if (object.nodeName !== Shape.IMAGE) { + return; + } + + imagePool.releaseImageRef(object); + } + renderingService.hooks.init.tap(LoadImagePlugin.tag, () => { canvas.addEventListener(ElementEvent.MOUNTED, handleMounted); canvas.addEventListener( ElementEvent.ATTR_MODIFIED, handleAttributeChanged, ); + canvas.addEventListener(ElementEvent.DESTROY, handleDestroy); }); renderingService.hooks.destroy.tap(LoadImagePlugin.tag, () => { @@ -85,6 +104,7 @@ export class LoadImagePlugin implements RenderingPlugin { ElementEvent.ATTR_MODIFIED, handleAttributeChanged, ); + canvas.removeEventListener(ElementEvent.DESTROY, handleDestroy); }); } } diff --git a/packages/g-plugin-image-loader/src/RefCountCache.ts b/packages/g-plugin-image-loader/src/RefCountCache.ts new file mode 100644 index 000000000..36c8c567e --- /dev/null +++ b/packages/g-plugin-image-loader/src/RefCountCache.ts @@ -0,0 +1,75 @@ +export class RefCountCache { + #cacheStore = new Map< + string, + { value: CacheValue; counter: Set } + >(); + + has(key: string) { + return this.#cacheStore.has(key); + } + + put(key: string, item: CacheValue, ref: CounterValue) { + if (this.#cacheStore.has(key)) { + return false; + } + + this.#cacheStore.set(key, { + value: item, + counter: new Set([ref]), + }); + + return true; + } + + get(key: string, ref: CounterValue) { + const cacheItem = this.#cacheStore.get(key); + if (!cacheItem) { + return null; + } + + cacheItem.counter.add(ref); + + return cacheItem.value; + } + + update(key: string, value: CacheValue, ref: CounterValue) { + const cacheItem = this.#cacheStore.get(key); + if (!cacheItem) { + return false; + } + + cacheItem.value = { ...cacheItem.value, ...value }; + cacheItem.counter.add(ref); + + return true; + } + + release(key: string, ref: CounterValue) { + const cacheItem = this.#cacheStore.get(key); + if (!cacheItem) { + return false; + } + + cacheItem.counter.delete(ref); + + if (cacheItem.counter.size <= 0) { + this.#cacheStore.delete(key); + } + + return true; + } + + releaseRef(ref: CounterValue) { + this.#cacheStore.keys().forEach((key) => { + this.release(key, ref); + }); + } + + getSize() { + return this.#cacheStore.size; + } + + clear() { + this.#cacheStore.clear(); + } +} diff --git a/packages/g-plugin-image-loader/src/index.ts b/packages/g-plugin-image-loader/src/index.ts index e406be1b2..d758da7c9 100644 --- a/packages/g-plugin-image-loader/src/index.ts +++ b/packages/g-plugin-image-loader/src/index.ts @@ -1,14 +1,14 @@ -import { AbstractRendererPlugin } from '@antv/g-lite'; -import { ImagePool } from './ImagePool'; +import { AbstractRendererPlugin, type GlobalRuntime } from '@antv/g-lite'; +import { ImagePool, type ImageCache } from './ImagePool'; import { LoadImagePlugin } from './LoadImagePlugin'; -export { ImagePool }; +export { ImagePool, type ImageCache }; export class Plugin extends AbstractRendererPlugin { name = 'image-loader'; - init(): void { + init(runtime: GlobalRuntime): void { // @ts-ignore - this.context.imagePool = new ImagePool(this.context.config); + this.context.imagePool = new ImagePool(this.context, runtime); this.addRenderingPlugin(new LoadImagePlugin()); } destroy(): void { diff --git a/rollup.config.mjs b/rollup.config.mjs index 046853d51..ad830668b 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -1,7 +1,6 @@ import commonjs from '@rollup/plugin-commonjs'; import nodeResolve from '@rollup/plugin-node-resolve'; import babel from '@rollup/plugin-babel'; -import typescript from '@rollup/plugin-typescript'; import terser from '@rollup/plugin-terser'; import replace from '@rollup/plugin-replace'; // import strip from '@rollup/plugin-strip'; @@ -9,7 +8,7 @@ import filesize from 'rollup-plugin-filesize'; // import { visualizer } from 'rollup-plugin-visualizer'; import { builtinModules } from 'module'; import process from 'node:process'; -import path from 'node:path'; +// import path from 'node:path'; // import { URL, fileURLToPath } from 'node:url'; // import fse from 'fs-extra'; import babelConfig from './babel.config.mjs'; @@ -75,7 +74,6 @@ export function createConfig({ sourceMap: enableSourceMap, }), ...plugins, - // typescript({ sourceMap: enableSourceMap }), // strip({ // include: ['src/**/*.(ts|tsx|js|jsx|mjs)'], // sourceMap: enableSourceMap,