diff --git a/packages/dev/core/src/Misc/decorators.ts b/packages/dev/core/src/Misc/decorators.ts index a38dc057e8f..4869fa30057 100644 --- a/packages/dev/core/src/Misc/decorators.ts +++ b/packages/dev/core/src/Misc/decorators.ts @@ -114,7 +114,7 @@ declare const _native: any; export function nativeOverride boolean>( target: any, propertyKey: string, - descriptor: TypedPropertyDescriptor<(...params: Parameters) => unknown>, + descriptor: TypedPropertyDescriptor<(...params: Parameters) => any>, predicate?: T ) { // Cache the original JS function for later. diff --git a/packages/dev/core/src/Misc/dumpTools.ts b/packages/dev/core/src/Misc/dumpTools.ts index 9d86eb7dc46..0d4464b3747 100644 --- a/packages/dev/core/src/Misc/dumpTools.ts +++ b/packages/dev/core/src/Misc/dumpTools.ts @@ -9,10 +9,12 @@ import { Clamp } from "../Maths/math.scalar.functions"; import type { AbstractEngine } from "../Engines/abstractEngine"; import { EngineStore } from "../Engines/engineStore"; import { Logger } from "./logger"; +import { EncodeArrayBufferToBase64 } from "./stringTools"; +import { nativeOverride } from "./decorators"; type DumpResources = { canvas: HTMLCanvasElement | OffscreenCanvas; - dumpEngine?: { + dumpEngine: { engine: ThinEngine; renderer: EffectRenderer; wrapper: EffectWrapper; @@ -28,14 +30,11 @@ async function _CreateDumpResourcesAsync(): Promise { Logger.Warn("DumpData: OffscreenCanvas will be used for dumping data. This may result in lossy alpha values."); } - // If WebGL via ThinEngine is not available (e.g. Native), use the BitmapRenderer. + // If WebGL via ThinEngine is not available, we cannot encode the data. // If https://github.com/whatwg/html/issues/10142 is resolved, we can migrate to just BitmapRenderer and avoid an engine dependency altogether. const { ThinEngine: thinEngineClass } = await import("../Engines/thinEngine"); if (!thinEngineClass.IsSupported) { - if (!canvas.getContext("bitmaprenderer")) { - throw new Error("DumpData: No WebGL or bitmap rendering context available. Cannot dump data."); - } - return { canvas }; + throw new Error("DumpData: No WebGL context available. Cannot dump data."); } const options = { @@ -85,6 +84,58 @@ async function _GetDumpResourcesAsync() { return await ResourcesPromise; } +class EncodingHelper { + /** + * Encodes image data to the given mime type. + * This is put into a helper class so we can apply the nativeOverride decorator to it. + * @internal + */ + @nativeOverride + public static async EncodeImageAsync(pixelData: ArrayBufferView, width: number, height: number, mimeType?: string, invertY?: boolean, quality?: number): Promise { + const resources = await _GetDumpResourcesAsync(); + + const dumpEngine = resources.dumpEngine; + dumpEngine.engine.setSize(width, height, true); + + // Create the image + const texture = dumpEngine.engine.createRawTexture(pixelData, width, height, Constants.TEXTUREFORMAT_RGBA, false, !invertY, Constants.TEXTURE_NEAREST_NEAREST); + + dumpEngine.renderer.setViewport(); + dumpEngine.renderer.applyEffectWrapper(dumpEngine.wrapper); + dumpEngine.wrapper.effect._bindTexture("textureSampler", texture); + dumpEngine.renderer.draw(); + + texture.dispose(); + + return await new Promise((resolve, reject) => { + Tools.ToBlob( + resources.canvas, + (blob) => { + if (!blob) { + reject(new Error("EncodeImageAsync: Failed to convert canvas to blob.")); + } else { + resolve(blob); + } + }, + mimeType, + quality + ); + }); + } +} + +/** + * Encodes pixel data to an image + * @param pixelData 8-bit RGBA pixel data + * @param width the width of the image + * @param height the height of the image + * @param mimeType the requested MIME type + * @param invertY true to invert the image in the Y direction + * @param quality the quality of the image if lossy mimeType is used (e.g. image/jpeg, image/webp). See {@link https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/toBlob | HTMLCanvasElement.toBlob()}'s `quality` parameter. + * @returns a promise that resolves to the encoded image data. Note that the `blob.type` may differ from `mimeType` if it was not supported. + */ +export const EncodeImageAsync = EncodingHelper.EncodeImageAsync; + /** * Dumps the current bound framebuffer * @param width defines the rendering width @@ -168,63 +219,23 @@ export async function DumpDataAsync( data = data2; } - const resources = await _GetDumpResourcesAsync(); + const blob = await EncodingHelper.EncodeImageAsync(data, width, height, mimeType, invertY, quality); - // Keep the async render + read from the shared canvas atomic - // eslint-disable-next-line no-async-promise-executor - return await new Promise(async (resolve) => { - if (resources.dumpEngine) { - const dumpEngine = resources.dumpEngine; - dumpEngine.engine.setSize(width, height, true); - - // Create the image - const texture = dumpEngine.engine.createRawTexture(data, width, height, Constants.TEXTUREFORMAT_RGBA, false, !invertY, Constants.TEXTURE_NEAREST_NEAREST); - - dumpEngine.renderer.setViewport(); - dumpEngine.renderer.applyEffectWrapper(dumpEngine.wrapper); - dumpEngine.wrapper.effect._bindTexture("textureSampler", texture); - dumpEngine.renderer.draw(); + if (fileName !== undefined) { + Tools.DownloadBlob(blob, fileName); + } - texture.dispose(); - } else { - const ctx = resources.canvas.getContext("bitmaprenderer") as ImageBitmapRenderingContext; - resources.canvas.width = width; - resources.canvas.height = height; + if (blob.type !== mimeType) { + Logger.Warn(`DumpData: The requested mimeType '${mimeType}' is not supported. The result has mimeType '${blob.type}' instead.`); + } - const imageData = new ImageData(width, height); // ImageData(data, sw, sh) ctor not yet widely implemented - imageData.data.set(data as Uint8ClampedArray); - const imageBitmap = await createImageBitmap(imageData, { premultiplyAlpha: "none", imageOrientation: invertY ? "flipY" : "from-image" }); + const buffer = await blob.arrayBuffer(); - ctx.transferFromImageBitmap(imageBitmap); - } + if (toArrayBuffer) { + return buffer; + } - Tools.ToBlob( - resources.canvas, - (blob) => { - if (!blob) { - throw new Error("DumpData: Failed to convert canvas to blob."); - } - - if (fileName !== undefined) { - Tools.DownloadBlob(blob, fileName); - } - - const fileReader = new FileReader(); - fileReader.onload = (event: any) => { - const result = event.target!.result as string | ArrayBuffer; - resolve(result); - }; - - if (toArrayBuffer) { - fileReader.readAsArrayBuffer(blob); - } else { - fileReader.readAsDataURL(blob); - } - }, - mimeType, - quality - ); - }); + return `data:${mimeType};base64,${EncodeArrayBufferToBase64(buffer)}`; } /** diff --git a/packages/dev/core/src/Misc/index.ts b/packages/dev/core/src/Misc/index.ts index 4296d137c0f..e816dec1069 100644 --- a/packages/dev/core/src/Misc/index.ts +++ b/packages/dev/core/src/Misc/index.ts @@ -70,8 +70,7 @@ export * from "./snapshotRenderingHelper"; // eslint-disable-next-line import/export export * from "./observableCoroutine"; export * from "./copyTextureToTexture"; -/** @deprecated Use individual exports */ -export { DumpTools } from "./dumpTools"; +export { DumpTools, EncodeImageAsync } from "./dumpTools"; export * from "./greasedLineTools"; export * from "./equirectangularCapture"; export * from "./decorators.serialization"; diff --git a/packages/dev/serializers/src/glTF/2.0/glTFExporter.ts b/packages/dev/serializers/src/glTF/2.0/glTFExporter.ts index 05bc3d89ec4..173fc4268b2 100644 --- a/packages/dev/serializers/src/glTF/2.0/glTFExporter.ts +++ b/packages/dev/serializers/src/glTF/2.0/glTFExporter.ts @@ -15,7 +15,6 @@ import type { ITextureInfo, ISkin, ICamera, - ImageMimeType, } from "babylonjs-gltf2interface"; import { AccessorComponentType, AccessorType, CameraType } from "babylonjs-gltf2interface"; import type { FloatArray, IndicesArray, Nullable } from "core/types"; @@ -243,7 +242,7 @@ export class GLTFExporter { public readonly _textures: ITexture[] = []; public readonly _babylonScene: Scene; - public readonly _imageData: { [fileName: string]: { data: ArrayBuffer; mimeType: ImageMimeType } } = {}; + public readonly _imageData: { [fileName: string]: Blob } = {}; /** * Baked animation sample rate @@ -561,7 +560,7 @@ export class GLTFExporter { if (this._imageData) { for (const image in this._imageData) { - container.files[image] = new Blob([this._imageData[image].data], { type: this._imageData[image].mimeType }); + container.files[image] = this._imageData[image]; } } diff --git a/packages/dev/serializers/src/glTF/2.0/glTFMaterialExporter.ts b/packages/dev/serializers/src/glTF/2.0/glTFMaterialExporter.ts index c25b565f30b..6bdea070368 100644 --- a/packages/dev/serializers/src/glTF/2.0/glTFMaterialExporter.ts +++ b/packages/dev/serializers/src/glTF/2.0/glTFMaterialExporter.ts @@ -18,7 +18,7 @@ import type { Scene } from "core/scene"; import type { GLTFExporter } from "./glTFExporter"; import { Constants } from "core/Engines/constants"; -import { DumpTools } from "core/Misc/dumpTools"; +import { EncodeImageAsync } from "core/Misc/dumpTools"; import type { Material } from "core/Materials/material"; import type { StandardMaterial } from "core/Materials/standardMaterial"; @@ -52,8 +52,8 @@ interface IPBRMetallicRoughness { baseColor: Color3; metallic: Nullable; roughness: Nullable; - metallicRoughnessTextureData?: Nullable; - baseColorTextureData?: Nullable; + metallicRoughnessTextureData?: Nullable; + baseColorTextureData?: Nullable; } function GetFileExtensionFromMimeType(mimeType: ImageMimeType): string { @@ -71,12 +71,29 @@ function GetFileExtensionFromMimeType(mimeType: ImageMimeType): string { } } +/** + * @param mimeType the MIME type requested by the user + * @returns true if the given mime type is compatible with glTF + */ +function IsSupportedMimeType(mimeType?: string): mimeType is ImageMimeType { + switch (mimeType) { + case ImageMimeType.JPEG: + case ImageMimeType.PNG: + case ImageMimeType.WEBP: + case ImageMimeType.AVIF: + case ImageMimeType.KTX2: + return true; + default: + return false; + } +} + /** * Gets cached image from a texture, if available. * @param babylonTexture texture to check for cached image * @returns image data if found and directly usable; null otherwise */ -async function GetCachedImageAsync(babylonTexture: BaseTexture): Promise> { +async function GetCachedImageAsync(babylonTexture: BaseTexture): Promise> { const internalTexture = babylonTexture.getInternalTexture(); if (!internalTexture || internalTexture.source !== InternalTextureSource.Url) { return null; @@ -108,8 +125,8 @@ async function GetCachedImageAsync(babylonTexture: BaseTexture): Promise { - return await DumpTools.DumpDataAsync(width, height, buffer, mimeType, undefined, false, true); - } - /** * Resizes the two source textures to the same dimensions. If a texture is null, a default white texture is generated. If both textures are null, returns null * @param texture1 first texture to resize @@ -370,14 +383,12 @@ export class GLTFMaterialExporter { * @param diffuseTexture texture used to store diffuse information * @param specularGlossinessTexture texture used to store specular and glossiness information * @param factors specular glossiness material factors - * @param mimeType the mime type to use for the texture * @returns pbr metallic roughness interface or null */ private async _convertSpecularGlossinessTexturesToMetallicRoughnessAsync( diffuseTexture: Nullable, specularGlossinessTexture: Nullable, - factors: IPBRSpecularGlossiness, - mimeType: ImageMimeType + factors: IPBRSpecularGlossiness ): Promise { const promises = new Array>(); if (!(diffuseTexture || specularGlossinessTexture)) { @@ -502,14 +513,14 @@ export class GLTFMaterialExporter { if (writeOutMetallicRoughnessTexture) { promises.push( - this._getImageDataAsync(metallicRoughnessBuffer, width, height, mimeType).then((data) => { + EncodeImageAsync(metallicRoughnessBuffer, width, height).then((data) => { metallicRoughnessFactors.metallicRoughnessTextureData = data; }) ); } if (writeOutBaseColorTexture) { promises.push( - this._getImageDataAsync(baseColorBuffer, width, height, mimeType).then((data) => { + EncodeImageAsync(baseColorBuffer, width, height).then((data) => { metallicRoughnessFactors.baseColorTextureData = data; }) ); @@ -816,7 +827,6 @@ export class GLTFMaterialExporter { pbrMetallicRoughness: IMaterialPbrMetallicRoughness, hasUVs: boolean ): Promise { - const mimeType = ImageMimeType.PNG; const specGloss: IPBRSpecularGlossiness = { diffuseColor: babylonPBRMaterial._albedoColor, specularColor: babylonPBRMaterial._reflectivityColor, @@ -834,17 +844,17 @@ export class GLTFMaterialExporter { this._exporter._materialNeedsUVsSet.add(babylonPBRMaterial); const samplerIndex = this._exportTextureSampler(albedoTexture || reflectivityTexture); - const metallicRoughnessFactors = await this._convertSpecularGlossinessTexturesToMetallicRoughnessAsync(albedoTexture, reflectivityTexture, specGloss, mimeType); + const metallicRoughnessFactors = await this._convertSpecularGlossinessTexturesToMetallicRoughnessAsync(albedoTexture, reflectivityTexture, specGloss); const textures = this._exporter._textures; if (metallicRoughnessFactors.baseColorTextureData) { - const imageIndex = this._exportImage(`baseColor${textures.length}`, mimeType, metallicRoughnessFactors.baseColorTextureData); + const imageIndex = await this._exportImageAsync(`baseColor${textures.length}`, metallicRoughnessFactors.baseColorTextureData); pbrMetallicRoughness.baseColorTexture = this._exportTextureInfo(imageIndex, samplerIndex, albedoTexture?.coordinatesIndex); } if (metallicRoughnessFactors.metallicRoughnessTextureData) { - const imageIndex = this._exportImage(`metallicRoughness${textures.length}`, mimeType, metallicRoughnessFactors.metallicRoughnessTextureData); + const imageIndex = await this._exportImageAsync(`metallicRoughness${textures.length}`, metallicRoughnessFactors.metallicRoughnessTextureData); pbrMetallicRoughness.metallicRoughnessTexture = this._exportTextureInfo(imageIndex, samplerIndex, reflectivityTexture?.coordinatesIndex); } @@ -1060,30 +1070,26 @@ export class GLTFMaterialExporter { imageIndexPromise = (async () => { // Try to get the image from memory first, if applicable const cache = await GetCachedImageAsync(babylonTexture); - if (cache && (requestedMimeType === "none" || cache.mimeType === requestedMimeType)) { - return this._exportImage(babylonTexture.name, cache.mimeType as ImageMimeType, cache.data); + if (cache && (requestedMimeType === "none" || cache.type === requestedMimeType)) { + return await this._exportImageAsync(babylonTexture.name, cache); } // Preserve texture mime type if defined let mimeType = ImageMimeType.PNG; if (requestedMimeType !== "none") { - switch (requestedMimeType) { - case ImageMimeType.JPEG: - case ImageMimeType.PNG: - case ImageMimeType.WEBP: - mimeType = requestedMimeType; - break; - default: - Tools.Warn(`Unsupported media type: ${requestedMimeType}. Exporting texture as PNG.`); - break; + if (IsSupportedMimeType(requestedMimeType)) { + mimeType = requestedMimeType; + } else { + mimeType = ImageMimeType.PNG; + Tools.Warn(`Unsupported media type: ${requestedMimeType}. Exporting texture as PNG.`); } } const size = babylonTexture.getSize(); const pixels = await GetTextureDataAsync(babylonTexture); - const data = await this._getImageDataAsync(pixels, size.width, size.height, mimeType); + const imageData = await EncodeImageAsync(pixels, size.width, size.height, mimeType); - return this._exportImage(babylonTexture.name, mimeType, data); + return await this._exportImageAsync(babylonTexture.name, imageData); })(); internalTextureToImage[internalTextureUniqueId][requestedMimeType] = imageIndexPromise; @@ -1092,22 +1098,23 @@ export class GLTFMaterialExporter { return await imageIndexPromise; } - private _exportImage(name: string, mimeType: ImageMimeType, data: ArrayBuffer): number { + private async _exportImageAsync(name: string, imageData: Blob): Promise { const images = this._exporter._images; let image: IImage; if (this._exporter._shouldUseGlb) { image = { name: name, - mimeType: mimeType, + mimeType: imageData.type as ImageMimeType, bufferView: undefined, // Will be updated later by BufferManager }; + const data = await imageData.arrayBuffer(); const bufferView = this._exporter._bufferManager.createBufferView(new Uint8Array(data)); this._exporter._bufferManager.setBufferView(image, bufferView); } else { // Build a unique URI const baseName = name.replace(/\.\/|\/|\.\\|\\/g, "_"); - const extension = GetFileExtensionFromMimeType(mimeType); + const extension = GetFileExtensionFromMimeType(imageData.type as ImageMimeType); let fileName = baseName + extension; if (images.some((image) => image.uri === fileName)) { fileName = `${baseName}_${Tools.RandomId()}${extension}`; @@ -1117,7 +1124,7 @@ export class GLTFMaterialExporter { name: name, uri: fileName, }; - this._exporter._imageData[fileName] = { data: data, mimeType: mimeType }; // Save image data to be written to file later + this._exporter._imageData[fileName] = imageData; // Save image data to be written to file later } images.push(image);