From af128d605a069150b75a6b466ab06ac5992e9c9d Mon Sep 17 00:00:00 2001 From: Evgeni Popov Date: Wed, 16 Nov 2022 15:15:05 +0100 Subject: [PATCH 1/2] Improve copy video to texture in WebGPU Former-commit-id: 319e63c604a77b3f4062453efce483c2c20ed108 --- .../Engines/Extensions/engine.videoTexture.ts | 3 +- .../WebGPU/Extensions/engine.videoTexture.ts | 41 ++-- .../src/Engines/WebGPU/webgpuTextureHelper.ts | 177 ++++++++++++++++++ .../Materials/Textures/htmlElementTexture.ts | 5 +- .../src/Materials/Textures/videoTexture.ts | 7 +- 5 files changed, 216 insertions(+), 17 deletions(-) diff --git a/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts b/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts index 05521910d86..7d921c8c726 100644 --- a/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts +++ b/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts @@ -2,6 +2,7 @@ import { ThinEngine } from "../../Engines/thinEngine"; import type { InternalTexture } from "../../Materials/Textures/internalTexture"; import type { Nullable } from "../../types"; import { Constants } from "../constants"; +import type { ExternalTexture } from "../..//Materials/Textures/externalTexture"; declare module "../../Engines/thinEngine" { export interface ThinEngine { @@ -11,7 +12,7 @@ declare module "../../Engines/thinEngine" { * @param video defines the video element to use * @param invertY defines if data must be stored with Y axis inverted */ - updateVideoTexture(texture: Nullable, video: HTMLVideoElement, invertY: boolean): void; + updateVideoTexture(texture: Nullable, video: HTMLVideoElement | Nullable, invertY: boolean): void; } } diff --git a/packages/dev/core/src/Engines/WebGPU/Extensions/engine.videoTexture.ts b/packages/dev/core/src/Engines/WebGPU/Extensions/engine.videoTexture.ts index 2422e865362..c4396760f1a 100644 --- a/packages/dev/core/src/Engines/WebGPU/Extensions/engine.videoTexture.ts +++ b/packages/dev/core/src/Engines/WebGPU/Extensions/engine.videoTexture.ts @@ -2,8 +2,13 @@ import type { InternalTexture } from "../../../Materials/Textures/internalTextur import type { Nullable } from "../../../types"; import { WebGPUEngine } from "../../webgpuEngine"; import type { WebGPUHardwareTexture } from "../webgpuHardwareTexture"; +import type { ExternalTexture } from "../../../Materials/Textures/externalTexture"; -WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable, video: HTMLVideoElement, invertY: boolean): void { +function IsExternalTexture(texture: Nullable | HTMLVideoElement): texture is ExternalTexture { + return texture && (texture as ExternalTexture).underlyingResource !== undefined ? true : false; +} + +WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable, video: HTMLVideoElement | Nullable, invertY: boolean): void { if (!texture || texture._isDisabled) { return; } @@ -18,18 +23,26 @@ WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable { - this._textureHelper.updateTexture(bitmap, texture, texture.width, texture.height, texture.depth, gpuTextureWrapper.format, 0, 0, !invertY, false, 0, 0); - if (texture.generateMipMaps) { - this._generateMipmaps(texture, this._uploadEncoder); - } + if (IsExternalTexture(video)) { + this._textureHelper.copyVideoToTexture(video, texture, gpuTextureWrapper.format, !invertY); + if (texture.generateMipMaps) { + this._generateMipmaps(texture, this._uploadEncoder); + } + texture.isReady = true; + } else if (video) { + this.createImageBitmap(video) + .then((bitmap) => { + this._textureHelper.updateTexture(bitmap, texture, texture.width, texture.height, texture.depth, gpuTextureWrapper.format, 0, 0, !invertY, false, 0, 0); + if (texture.generateMipMaps) { + this._generateMipmaps(texture, this._uploadEncoder); + } - texture.isReady = true; - }) - .catch(() => { - // Sometimes createImageBitmap(video) fails with "Failed to execute 'createImageBitmap' on 'Window': The provided element's player has no current data." - // Just keep going on - texture.isReady = true; - }); + texture.isReady = true; + }) + .catch(() => { + // Sometimes createImageBitmap(video) fails with "Failed to execute 'createImageBitmap' on 'Window': The provided element's player has no current data." + // Just keep going on + texture.isReady = true; + }); + } }; diff --git a/packages/dev/core/src/Engines/WebGPU/webgpuTextureHelper.ts b/packages/dev/core/src/Engines/WebGPU/webgpuTextureHelper.ts index dbeafa90dc6..02826c33649 100644 --- a/packages/dev/core/src/Engines/WebGPU/webgpuTextureHelper.ts +++ b/packages/dev/core/src/Engines/WebGPU/webgpuTextureHelper.ts @@ -31,6 +31,7 @@ import type { HardwareTextureWrapper } from "../../Materials/Textures/hardwareTe import type { BaseTexture } from "../../Materials/Textures/baseTexture"; import { WebGPUHardwareTexture } from "./webgpuHardwareTexture"; import type { WebGPUTintWASM } from "./webgpuTintWASM"; +import type { ExternalTexture } from "../../Materials/Textures/externalTexture"; // TODO WEBGPU improve mipmap generation by using compute shaders @@ -159,6 +160,62 @@ const clearFragmentSource = ` } `; +const copyVideoToTextureVertexSource = ` + struct VertexOutput { + @builtin(position) Position : vec4, + @location(0) fragUV : vec2 + } + + @vertex + fn main( + @builtin(vertex_index) VertexIndex : u32 + ) -> VertexOutput { + var pos = array, 4>( + vec2(-1.0, 1.0), + vec2( 1.0, 1.0), + vec2(-1.0, -1.0), + vec2( 1.0, -1.0) + ); + var tex = array, 4>( + vec2(0.0, 0.0), + vec2(1.0, 0.0), + vec2(0.0, 1.0), + vec2(1.0, 1.0) + ); + + var output: VertexOutput; + + output.Position = vec4(pos[VertexIndex], 0.0, 1.0); + output.fragUV = tex[VertexIndex]; + + return output; + } + `; + +const copyVideoToTextureFragmentSource = ` + @group(0) @binding(0) var videoSampler: sampler; + @group(0) @binding(1) var videoTexture: texture_external; + + @fragment + fn main( + @location(0) fragUV: vec2 + ) -> @location(0) vec4 { + return textureSampleBaseClampToEdge(videoTexture, videoSampler, fragUV); + } + `; + +const copyVideoToTextureInvertYFragmentSource = ` + @group(0) @binding(0) var videoSampler: sampler; + @group(0) @binding(1) var videoTexture: texture_external; + + @fragment + fn main( + @location(0) fragUV: vec2 + ) -> @location(0) vec4 { + return textureSampleBaseClampToEdge(videoTexture, videoSampler, vec2(fragUV.x, 1.0 - fragUV.y)); + } + `; + enum PipelineType { MipMap = 0, InvertYPremultiplyAlpha = 1, @@ -166,6 +223,11 @@ enum PipelineType { InvertYPremultiplyAlphaWithOfst = 3, } +enum VideoPipelineType { + DontInvertY = 0, + InvertY = 1, +} + interface IPipelineParameters { invertY?: boolean; premultiplyAlpha?: boolean; @@ -238,9 +300,12 @@ export class WebGPUTextureHelper { private _tintWASM: Nullable; private _bufferManager: WebGPUBufferManager; private _mipmapSampler: GPUSampler; + private _videoSampler: GPUSampler; private _ubCopyWithOfst: GPUBuffer; private _pipelines: { [format: string]: Array<[GPURenderPipeline, GPUBindGroupLayout]> } = {}; private _compiledShaders: GPUShaderModule[][] = []; + private _videoPipelines: { [format: string]: Array<[GPURenderPipeline, GPUBindGroupLayout]> } = {}; + private _videoCompiledShaders: GPUShaderModule[][] = []; private _deferredReleaseTextures: Array<[Nullable, Nullable]> = []; private _commandEncoderForCreation: GPUCommandEncoder; @@ -259,9 +324,11 @@ export class WebGPUTextureHelper { this._bufferManager = bufferManager; this._mipmapSampler = device.createSampler({ minFilter: WebGPUConstants.FilterMode.Linear }); + this._videoSampler = device.createSampler({ minFilter: WebGPUConstants.FilterMode.Linear }); this._ubCopyWithOfst = this._bufferManager.createBuffer(4 * 4, WebGPUConstants.BufferUsage.Uniform | WebGPUConstants.BufferUsage.CopyDst).underlyingResource; this._getPipeline(WebGPUConstants.TextureFormat.RGBA8Unorm); + this._getVideoPipeline(WebGPUConstants.TextureFormat.RGBA8Unorm); } private _getPipeline(format: GPUTextureFormat, type: PipelineType = PipelineType.MipMap, params?: IPipelineParameters): [GPURenderPipeline, GPUBindGroupLayout] { @@ -338,6 +405,54 @@ export class WebGPUTextureHelper { return pipelineAndBGL; } + private _getVideoPipeline(format: GPUTextureFormat, type: VideoPipelineType = VideoPipelineType.DontInvertY): [GPURenderPipeline, GPUBindGroupLayout] { + const index = type === VideoPipelineType.InvertY ? 1 << 0 : 0; + + if (!this._videoPipelines[format]) { + this._videoPipelines[format] = []; + } + + let pipelineAndBGL = this._videoPipelines[format][index]; + if (!pipelineAndBGL) { + let modules = this._videoCompiledShaders[index]; + if (!modules) { + const vertexModule = this._device.createShaderModule({ + code: copyVideoToTextureVertexSource, + }); + const fragmentModule = this._device.createShaderModule({ + code: index === 0 ? copyVideoToTextureFragmentSource : copyVideoToTextureInvertYFragmentSource, + }); + modules = this._videoCompiledShaders[index] = [vertexModule, fragmentModule]; + } + + const pipeline = this._device.createRenderPipeline({ + label: `CopyVideoToTexture_${format}_${index === 0 ? "DontInvertY" : "InvertY"}`, + layout: WebGPUConstants.AutoLayoutMode.Auto, + vertex: { + module: modules[0], + entryPoint: "main", + }, + fragment: { + module: modules[1], + entryPoint: "main", + targets: [ + { + format, + }, + ], + }, + primitive: { + topology: WebGPUConstants.PrimitiveTopology.TriangleStrip, + stripIndexFormat: WebGPUConstants.IndexFormat.Uint16, + }, + }); + + pipelineAndBGL = this._videoPipelines[format][index] = [pipeline, pipeline.getBindGroupLayout(0)]; + } + + return pipelineAndBGL; + } + private static _GetTextureTypeFromFormat(format: GPUTextureFormat): number { switch (format) { // One Component = 8 bits @@ -1010,6 +1125,68 @@ export class WebGPUTextureHelper { return false; } + public copyVideoToTexture(video: ExternalTexture, texture: InternalTexture, format: GPUTextureFormat, invertY = false, commandEncoder?: GPUCommandEncoder): void { + const useOwnCommandEncoder = commandEncoder === undefined; + const [pipeline, bindGroupLayout] = this._getVideoPipeline(format, invertY ? VideoPipelineType.InvertY : VideoPipelineType.DontInvertY); + + if (useOwnCommandEncoder) { + commandEncoder = this._device.createCommandEncoder({}); + } + + commandEncoder!.pushDebugGroup?.(`copy video to texture - invertY=${invertY}`); + + const webgpuHardwareTexture = texture._hardwareTexture as WebGPUHardwareTexture; + + const renderPassDescriptor: GPURenderPassDescriptor = { + colorAttachments: [ + { + view: webgpuHardwareTexture.underlyingResource!.createView({ + format, + dimension: WebGPUConstants.TextureViewDimension.E2d, + mipLevelCount: 1, + baseArrayLayer: 0, + baseMipLevel: 0, + arrayLayerCount: 1, + aspect: WebGPUConstants.TextureAspect.All, + }), + loadOp: WebGPUConstants.LoadOp.Load, + storeOp: WebGPUConstants.StoreOp.Store, + }, + ], + }; + const passEncoder = commandEncoder!.beginRenderPass(renderPassDescriptor); + + const descriptor: GPUBindGroupDescriptor = { + layout: bindGroupLayout, + entries: [ + { + binding: 0, + resource: this._videoSampler, + }, + { + binding: 1, + resource: this._device.importExternalTexture({ + source: video.underlyingResource, + }), + }, + ], + }; + + const bindGroup = this._device.createBindGroup(descriptor); + + passEncoder.setPipeline(pipeline); + passEncoder.setBindGroup(0, bindGroup); + passEncoder.draw(4, 1, 0, 0); + passEncoder.end(); + + commandEncoder!.popDebugGroup?.(); + + if (useOwnCommandEncoder) { + this._device.queue.submit([commandEncoder!.finish()]); + commandEncoder = null as any; + } + } + public invertYPreMultiplyAlpha( gpuOrHdwTexture: GPUTexture | WebGPUHardwareTexture, width: number, diff --git a/packages/dev/core/src/Materials/Textures/htmlElementTexture.ts b/packages/dev/core/src/Materials/Textures/htmlElementTexture.ts index 83c1aed6547..0de657368d7 100644 --- a/packages/dev/core/src/Materials/Textures/htmlElementTexture.ts +++ b/packages/dev/core/src/Materials/Textures/htmlElementTexture.ts @@ -3,6 +3,7 @@ import { BaseTexture } from "../../Materials/Textures/baseTexture"; import { Constants } from "../../Engines/constants"; import { Matrix } from "../../Maths/math.vector"; import { Observable } from "../../Misc/observable"; +import type { ExternalTexture } from "./externalTexture"; import "../../Engines/Extensions/engine.dynamicTexture"; import "../../Engines/Extensions/engine.videoTexture"; @@ -69,6 +70,7 @@ export class HtmlElementTexture extends BaseTexture { private _isVideo: boolean; private _generateMipMaps: boolean; private _samplingMode: number; + private _externalTexture: Nullable; /** * Instantiates a HtmlElementTexture from the following parameters. @@ -97,6 +99,7 @@ export class HtmlElementTexture extends BaseTexture { this.name = name; this.element = element; this._isVideo = element instanceof HTMLVideoElement; + this._externalTexture = this._isVideo ? this._engine?.createExternalTexture(element as HTMLVideoElement) ?? null : null; this.anisotropicFilteringLevel = 1; @@ -147,7 +150,7 @@ export class HtmlElementTexture extends BaseTexture { return; } - engine.updateVideoTexture(this._texture, videoElement, invertY === null ? true : invertY); + engine.updateVideoTexture(this._texture, this._externalTexture ? this._externalTexture : videoElement, invertY === null ? true : invertY); } else { const canvasElement = this.element as HTMLCanvasElement; engine.updateDynamicTexture(this._texture, canvasElement, invertY === null ? true : invertY, false, this._format); diff --git a/packages/dev/core/src/Materials/Textures/videoTexture.ts b/packages/dev/core/src/Materials/Textures/videoTexture.ts index c50496d039a..a1ec98aee78 100644 --- a/packages/dev/core/src/Materials/Textures/videoTexture.ts +++ b/packages/dev/core/src/Materials/Textures/videoTexture.ts @@ -5,6 +5,7 @@ import type { Nullable } from "../../types"; import type { Scene } from "../../scene"; import { Texture } from "../../Materials/Textures/texture"; import { Constants } from "../../Engines/constants"; +import type { ExternalTexture } from "./externalTexture"; import "../../Engines/Extensions/engine.videoTexture"; import "../../Engines/Extensions/engine.dynamicTexture"; @@ -76,6 +77,7 @@ export class VideoTexture extends Texture { */ public readonly video: HTMLVideoElement; + private _externalTexture: Nullable; private _onUserActionRequestedObservable: Nullable> = null; /** @@ -174,6 +176,7 @@ export class VideoTexture extends Texture { this._currentSrc = src; this.name = name || this._getName(src); this.video = this._getVideo(src); + this._externalTexture = this._engine?.createExternalTexture(this.video) ?? null; if (this._settings.poster) { this.video.poster = this._settings.poster; @@ -367,7 +370,7 @@ export class VideoTexture extends Texture { this._frameId = frameId; - this._getEngine()!.updateVideoTexture(this._texture, this.video, this._invertY); + this._getEngine()!.updateVideoTexture(this._texture, this._externalTexture ? this._externalTexture : this.video, this._invertY); }; /** @@ -405,6 +408,8 @@ export class VideoTexture extends Texture { this.video.removeEventListener("seeked", this._updateInternalTexture); this.video.removeEventListener("emptied", this._reset); this.video.pause(); + + this._externalTexture?.dispose(); } /** From a19c1ae3d944ef2f86da6684b701771a70a83134 Mon Sep 17 00:00:00 2001 From: Evgeni Popov Date: Wed, 16 Nov 2022 15:17:39 +0100 Subject: [PATCH 2/2] Fix typo Former-commit-id: 068e351487bbd4ff0433ae8d43a47e830b561cd0 --- packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts b/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts index 7d921c8c726..63fab673e51 100644 --- a/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts +++ b/packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts @@ -2,7 +2,7 @@ import { ThinEngine } from "../../Engines/thinEngine"; import type { InternalTexture } from "../../Materials/Textures/internalTexture"; import type { Nullable } from "../../types"; import { Constants } from "../constants"; -import type { ExternalTexture } from "../..//Materials/Textures/externalTexture"; +import type { ExternalTexture } from "../../Materials/Textures/externalTexture"; declare module "../../Engines/thinEngine" { export interface ThinEngine {