From 05e33acdfb8739b7389900990d6b8978480d4f17 Mon Sep 17 00:00:00 2001 From: xiaoiver Date: Tue, 2 Jan 2024 20:41:37 +0800 Subject: [PATCH] Fix readpixel (#130) * fix: read texture in webgpu * chore: commit changeset --- .changeset/late-moons-help.md | 5 + examples/demos/canvas-gradient.ts | 291 ++++++++++++++++++++++++++++ examples/demos/index.ts | 1 + examples/demos/read-pixel.ts | 8 +- examples/demos/render-to-texture.ts | 6 +- examples/utils/gradient.ts | 26 +++ src/webgpu/Readback.ts | 54 +++--- 7 files changed, 364 insertions(+), 27 deletions(-) create mode 100644 .changeset/late-moons-help.md create mode 100644 examples/demos/canvas-gradient.ts create mode 100644 examples/utils/gradient.ts diff --git a/.changeset/late-moons-help.md b/.changeset/late-moons-help.md new file mode 100644 index 0000000..b1d0345 --- /dev/null +++ b/.changeset/late-moons-help.md @@ -0,0 +1,5 @@ +--- +'@antv/g-device-api': patch +--- + +Read texture in webgpu. diff --git a/examples/demos/canvas-gradient.ts b/examples/demos/canvas-gradient.ts new file mode 100644 index 0000000..85961d3 --- /dev/null +++ b/examples/demos/canvas-gradient.ts @@ -0,0 +1,291 @@ +import { + DeviceContribution, + VertexStepMode, + Format, + TransparentWhite, + BufferUsage, + BufferFrequencyHint, + BlendMode, + BlendFactor, + TextureUsage, + CullMode, + ChannelWriteMask, + TransparentBlack, + CompareFunction, + AddressMode, + FilterMode, + MipmapFilterMode, +} from '../../src'; +import { vec3, mat4 } from 'gl-matrix'; +import { + cubeVertexArray, + cubeVertexSize, + cubeVertexCount, + cubePositionOffset, + cubeUVOffset, +} from '../meshes/cube'; +import { generateColorRamp } from '../utils/gradient'; + +export async function render( + deviceContribution: DeviceContribution, + $canvas: HTMLCanvasElement, + useRAF = true, + image?: HTMLImageElement, +) { + // create swap chain and get device + const swapChain = await deviceContribution.createSwapChain($canvas); + + // TODO: resize + swapChain.configureSwapChain($canvas.width, $canvas.height); + const device = swapChain.getDevice(); + + const ramp = generateColorRamp({ + colors: [ + '#FF4818', + '#F7B74A', + '#FFF598', + '#91EABC', + '#2EA9A1', + '#206C7C', + ].reverse(), + positions: [0, 0.2, 0.4, 0.6, 0.8, 1.0], + }); + const gradientTexture = device.createTexture({ + format: Format.U8_RGBA_NORM, + width: ramp.width, + height: ramp.height, + usage: TextureUsage.SAMPLED, + }); + device.setResourceName(gradientTexture, 'Gradient Texture'); + gradientTexture.setImageData([ramp.data]); + + const program = device.createProgram({ + vertex: { + glsl: ` + layout(std140) uniform Uniforms { + mat4 u_ModelViewProjectionMatrix; + float u_Test; + }; + + layout(location = 0) in vec4 a_Position; + layout(location = 1) in vec2 a_Uv; + + out vec2 v_Uv; + + void main() { + v_Uv = a_Uv; + gl_Position = u_ModelViewProjectionMatrix * a_Position; + } + `, + }, + fragment: { + glsl: ` + uniform sampler2D u_Texture; + in vec2 v_Uv; + out vec4 outputColor; + + void main() { + outputColor = texture(SAMPLER_2D(u_Texture), v_Uv); + } + `, + }, + }); + + const vertexBuffer = device.createBuffer({ + viewOrSize: cubeVertexArray, + usage: BufferUsage.VERTEX, + }); + + const uniformBuffer = device.createBuffer({ + viewOrSize: 16 * 4 + 4 * 4, // mat4 + usage: BufferUsage.UNIFORM, + hint: BufferFrequencyHint.DYNAMIC, + }); + + const sampler = device.createSampler({ + addressModeU: AddressMode.CLAMP_TO_EDGE, + addressModeV: AddressMode.CLAMP_TO_EDGE, + minFilter: FilterMode.POINT, + magFilter: FilterMode.BILINEAR, + mipmapFilter: MipmapFilterMode.LINEAR, + lodMinClamp: 0, + lodMaxClamp: 0, + }); + + const inputLayout = device.createInputLayout({ + vertexBufferDescriptors: [ + { + arrayStride: cubeVertexSize, + stepMode: VertexStepMode.VERTEX, + attributes: [ + { + shaderLocation: 0, + offset: cubePositionOffset, + format: Format.F32_RGBA, + }, + { + shaderLocation: 1, + offset: cubeUVOffset, + format: Format.F32_RG, + }, + ], + }, + ], + indexBufferFormat: null, + program, + }); + + const pipeline = device.createRenderPipeline({ + inputLayout, + program, + colorAttachmentFormats: [Format.U8_RGBA_RT], + depthStencilAttachmentFormat: Format.D24_S8, + megaStateDescriptor: { + attachmentsState: [ + { + channelWriteMask: ChannelWriteMask.ALL, + rgbBlendState: { + blendMode: BlendMode.ADD, + blendSrcFactor: BlendFactor.SRC_ALPHA, + blendDstFactor: BlendFactor.ONE_MINUS_SRC_ALPHA, + }, + alphaBlendState: { + blendMode: BlendMode.ADD, + blendSrcFactor: BlendFactor.ONE, + blendDstFactor: BlendFactor.ONE_MINUS_SRC_ALPHA, + }, + }, + ], + blendConstant: TransparentBlack, + depthWrite: true, + depthCompare: CompareFunction.LESS, + cullMode: CullMode.BACK, + stencilWrite: false, + }, + }); + + const bindings = device.createBindings({ + pipeline, + uniformBufferBindings: [ + { + binding: 0, + buffer: uniformBuffer, + }, + ], + samplerBindings: [ + { + texture: gradientTexture, + sampler, + }, + ], + }); + + const mainColorRT = device.createRenderTargetFromTexture( + device.createTexture({ + format: Format.U8_RGBA_RT, + width: $canvas.width, + height: $canvas.height, + usage: TextureUsage.RENDER_TARGET, + }), + ); + const mainDepthRT = device.createRenderTargetFromTexture( + device.createTexture({ + format: Format.D24_S8, + width: $canvas.width, + height: $canvas.height, + usage: TextureUsage.RENDER_TARGET, + }), + ); + + let id: number; + const frame = () => { + const aspect = $canvas.width / $canvas.height; + const projectionMatrix = mat4.perspective( + mat4.create(), + (2 * Math.PI) / 5, + aspect, + 0.1, + 1000, + ); + const viewMatrix = mat4.identity(mat4.create()); + const modelViewProjectionMatrix = mat4.create(); + mat4.translate(viewMatrix, viewMatrix, vec3.fromValues(0, 0, -4)); + const now = useRAF ? Date.now() / 1000 : 0; + mat4.rotate( + viewMatrix, + viewMatrix, + 1, + vec3.fromValues(Math.sin(now), Math.cos(now), 0), + ); + mat4.multiply(modelViewProjectionMatrix, projectionMatrix, viewMatrix); + uniformBuffer.setSubData( + 0, + new Uint8Array((modelViewProjectionMatrix as Float32Array).buffer), + ); + // WebGL1 need this + program.setUniformsLegacy({ + u_ModelViewProjectionMatrix: modelViewProjectionMatrix, + u_Texture: gradientTexture, + }); + + /** + * An application should call getCurrentTexture() in the same task that renders to the canvas texture. + * Otherwise, the texture could get destroyed by these steps before the application is finished rendering to it. + */ + const onscreenTexture = swapChain.getOnscreenTexture(); + + const renderPass = device.createRenderPass({ + colorAttachment: [mainColorRT], + colorResolveTo: [onscreenTexture], + colorClearColor: [TransparentWhite], + depthStencilAttachment: mainDepthRT, + depthClearValue: 1, + }); + + renderPass.setPipeline(pipeline); + renderPass.setVertexInput( + inputLayout, + [ + { + buffer: vertexBuffer, + }, + ], + null, + ); + renderPass.setViewport(0, 0, $canvas.width, $canvas.height); + renderPass.setBindings(bindings); + renderPass.draw(cubeVertexCount); + + device.submitPass(renderPass); + if (useRAF) { + id = requestAnimationFrame(frame); + } + }; + + frame(); + + return () => { + if (useRAF && id) { + cancelAnimationFrame(id); + } + program.destroy(); + vertexBuffer.destroy(); + uniformBuffer.destroy(); + inputLayout.destroy(); + bindings.destroy(); + pipeline.destroy(); + mainColorRT.destroy(); + mainDepthRT.destroy(); + + sampler.destroy(); + device.destroy(); + + // For debug. + device.checkForLeaks(); + }; +} + +render.params = { + targets: ['webgl1', 'webgl2', 'webgpu'], + default: 'webgpu', +}; diff --git a/examples/demos/index.ts b/examples/demos/index.ts index af44a25..41aed3d 100644 --- a/examples/demos/index.ts +++ b/examples/demos/index.ts @@ -6,6 +6,7 @@ export { render as MSAA } from './msaa'; export { render as MultipleRenderTargets } from './multiple-render-targets'; export { render as MultipleRenderPasses } from './multiple-render-passes'; export { render as Blit } from './blit'; +export { render as CanvasGradient } from './canvas-gradient'; export { render as RotatingCube } from './rotating-cube'; export { render as NestedRenderPass } from './nested-render-pass'; export { render as Stencil } from './stencil'; diff --git a/examples/demos/read-pixel.ts b/examples/demos/read-pixel.ts index e2b1284..f09167b 100644 --- a/examples/demos/read-pixel.ts +++ b/examples/demos/read-pixel.ts @@ -136,7 +136,13 @@ void main() { const row = width * 4; const end = (height - 1) * row; for (let i = 0; i < length; i += row) { - ci.data.set(data.subarray(i, i + row), i); + const r = data.subarray(i, i + row); // bgra + // for (let j = 0; j < row; j += 4) { + // const t = r[j]; + // r[j] = r[j + 2]; + // r[j + 2] = t; + // } + ci.data.set(r, i); } context.putImageData(ci, 0, 0); diff --git a/examples/demos/render-to-texture.ts b/examples/demos/render-to-texture.ts index 944761a..df0111b 100644 --- a/examples/demos/render-to-texture.ts +++ b/examples/demos/render-to-texture.ts @@ -86,15 +86,16 @@ void main() { const trianglePipeline = device.createRenderPipeline({ inputLayout: triangleInputLayout, program: triangleProgram, - colorAttachmentFormats: [Format.U8_RGBA_RT], + colorAttachmentFormats: [Format.U8_RGBA_NORM], }); const triangleTexture = device.createTexture({ - format: Format.U8_RGBA_RT, + format: Format.U8_RGBA_NORM, width: $canvas.width, height: $canvas.height, usage: TextureUsage.RENDER_TARGET, }); + device.setResourceName(triangleTexture, 'Triangle Texture'); const triangleRenderTarget = device.createRenderTargetFromTexture(triangleTexture); @@ -277,6 +278,7 @@ void main() { colorAttachment: [triangleRenderTarget], colorResolveTo: [null], colorClearColor: [TransparentWhite], + colorStore: [true], depthStencilAttachment: null, depthStencilResolveTo: null, }); diff --git a/examples/utils/gradient.ts b/examples/utils/gradient.ts new file mode 100644 index 0000000..cef7447 --- /dev/null +++ b/examples/utils/gradient.ts @@ -0,0 +1,26 @@ +export function generateColorRamp(colorRamp: any): any { + let canvas = window.document.createElement('canvas'); + let ctx = canvas.getContext('2d') as CanvasRenderingContext2D; + canvas.width = 256; + canvas.height = 1; + let data: Uint8ClampedArray | undefined = undefined; + + // draw linear color + const gradient = ctx.createLinearGradient(0, 0, 256, 1); + + const min = colorRamp.positions[0]; + const max = colorRamp.positions[colorRamp.positions.length - 1]; + for (let i = 0; i < colorRamp.colors.length; ++i) { + const value = (colorRamp.positions[i] - min) / (max - min); + gradient.addColorStop(value, colorRamp.colors[i]); + } + ctx.fillStyle = gradient; + ctx.fillRect(0, 0, 256, 1); + + data = new Uint8ClampedArray(ctx.getImageData(0, 0, 256, 1).data); + // @ts-ignore + canvas = null; + // @ts-ignore + ctx = null; + return { data, width: 256, height: 1 }; +} diff --git a/src/webgpu/Readback.ts b/src/webgpu/Readback.ts index 412d924..0c87afc 100644 --- a/src/webgpu/Readback.ts +++ b/src/webgpu/Readback.ts @@ -87,6 +87,11 @@ export class Readback_WebGPU extends ResourceBase_WebGPU implements Readback { dstOffset, size, texture.format, + true, + false, + bytesPerRow, + bytesPerRowAligned, + height, ); } @@ -112,8 +117,9 @@ export class Readback_WebGPU extends ResourceBase_WebGPU implements Readback { type: Format = Format.U8_RGB, noDataConversion = false, destroy = false, - // bytesPerRow: number, - // bytesPerRowAligned: number, + bytesPerRow = 0, + bytesPerRowAligned = 0, + height = 0, ): Promise { const buffer = b as Buffer_WebGPU; @@ -227,28 +233,28 @@ export class Readback_WebGPU extends ResourceBase_WebGPU implements Readback { } } } - // if (bytesPerRow !== bytesPerRowAligned) { - // // TODO WEBGPU use computer shaders (or render pass) to build the final buffer data? - // if (floatFormat === 1 && !noDataConversion) { - // // half float have been converted to float above - // bytesPerRow *= 2; - // bytesPerRowAligned *= 2; - // } - // const data2 = new Uint8Array(data!.buffer); - // let offset = bytesPerRow, - // offset2 = 0; - // for (let y = 1; y < height; ++y) { - // offset2 = y * bytesPerRowAligned; - // for (let x = 0; x < bytesPerRow; ++x) { - // data2[offset++] = data2[offset2++]; - // } - // } - // if (floatFormat !== 0 && !noDataConversion) { - // data = new Float32Array(data2.buffer, 0, offset / 4); - // } else { - // data = new Uint8Array(data2.buffer, 0, offset); - // } - // } + if (bytesPerRow !== bytesPerRowAligned) { + // TODO WEBGPU use computer shaders (or render pass) to build the final buffer data? + if (floatFormat === 1 && !noDataConversion) { + // half float have been converted to float above + bytesPerRow *= 2; + bytesPerRowAligned *= 2; + } + const data2 = new Uint8Array(data!.buffer); + let offset = bytesPerRow, + offset2 = 0; + for (let y = 1; y < height; ++y) { + offset2 = y * bytesPerRowAligned; + for (let x = 0; x < bytesPerRow; ++x) { + data2[offset++] = data2[offset2++]; + } + } + if (floatFormat !== 0 && !noDataConversion) { + data = new Float32Array(data2.buffer, 0, offset / 4); + } else { + data = new Uint8Array(data2.buffer, 0, offset); + } + } gpuReadBuffer.gpuBuffer.unmap(); resolve(data!);