WIP - dither transparency with TAA (#6159)

* WIP - dither transparency with TAA

* lint

---------

Co-authored-by: Martin Valigursky <mvaligursky@snapchat.com>

examples/src/examples/graphics/dithered-transparency/controls.mjs

@@ -48,6 +48,15 @@ export function controls({ observer, ReactPCUI, React, jsx, fragment }) {
                         { v: pc.DITHER_BLUENOISE, t: 'BlueNoise' }
                     ]
                 })
+            ),
+            jsx(
+                LabelGroup,
+                { text: 'TAA (WIP)' },
+                jsx(BooleanInput, {
+                    type: 'toggle',
+                    binding: new BindingTwoWay(),
+                    link: { observer, path: 'data.taa' }
+                })
             )
         )
     );

examples/src/examples/graphics/dithered-transparency/example.mjs

@@ -15,13 +15,17 @@ const assets = {
         { type: pc.TEXTURETYPE_RGBP, mipmaps: false }
     ),
     table: new pc.Asset('table', 'container', { url: rootPath + '/static/assets/models/glass-table.glb' }),
-    script: new pc.Asset('script', 'script', { url: rootPath + '/static/scripts/camera/orbit-camera.js' })
+    script: new pc.Asset('script', 'script', { url: rootPath + '/static/scripts/camera/orbit-camera.js' }),
+    diffuse: new pc.Asset('color', 'texture', { url: rootPath + '/static/assets/textures/seaside-rocks01-color.jpg' }),
 };
 
 const gfxOptions = {
     deviceTypes: [deviceType],
     glslangUrl: rootPath + '/static/lib/glslang/glslang.js',
-    twgslUrl: rootPath + '/static/lib/twgsl/twgsl.js'
+    twgslUrl: rootPath + '/static/lib/twgsl/twgsl.js',
+
+    // disable anti-aliasing as TAA is used to smooth edges
+    antialias: false
 };
 
 const device = await pc.createGraphicsDevice(canvas, gfxOptions);
@@ -78,6 +82,7 @@ assetListLoader.load(() => {
         // create material of specified color
         const material = new pc.StandardMaterial();
         material.diffuse = color;
+        material.diffuseMap = assets.diffuse.resource;
         material.update();
 
         // create primitive
@@ -113,31 +118,6 @@ assetListLoader.load(() => {
         });
     });
 
-    // Create the camera
-    const camera = new pc.Entity('Camera');
-    camera.addComponent('camera', {
-        fov: 70
-    });
-    camera.translate(-14, 12, 12);
-    camera.lookAt(1, 4, 0);
-    app.root.addChild(camera);
-
-    // enable the camera to render the scene's color map, as the table material needs it
-    camera.camera.requestSceneColorMap(true);
-
-    // add orbit camera script with a mouse and a touch support
-    camera.addComponent('script');
-    camera.script.create('orbitCamera', {
-        attributes: {
-            inertiaFactor: 0.2,
-            focusEntity: tableEntity,
-            distanceMax: 30,
-            frameOnStart: false
-        }
-    });
-    camera.script.create('orbitCameraInputMouse');
-    camera.script.create('orbitCameraInputTouch');
-
     // Create an Entity with a directional light, casting soft VSM shadow
     const light = new pc.Entity();
     light.addComponent('light', {
@@ -154,6 +134,77 @@ assetListLoader.load(() => {
     light.setLocalEulerAngles(75, 120, 20);
     app.root.addChild(light);
 
+    // Create the camera
+    const cameraEntity = new pc.Entity('Camera');
+    cameraEntity.addComponent('camera', {
+        fov: 70
+    });
+    cameraEntity.translate(-14, 12, 12);
+    cameraEntity.lookAt(1, 4, 0);
+    app.root.addChild(cameraEntity);
+
+    // enable the camera to render the scene's color map, as the table material needs it
+    cameraEntity.camera.requestSceneColorMap(true);
+
+    // add orbit camera script with a mouse and a touch support
+    cameraEntity.addComponent('script');
+    cameraEntity.script.create('orbitCamera', {
+        attributes: {
+            inertiaFactor: 0.2,
+            focusEntity: tableEntity,
+            distanceMax: 30,
+            frameOnStart: false
+        }
+    });
+    cameraEntity.script.create('orbitCameraInputMouse');
+    cameraEntity.script.create('orbitCameraInputTouch');
+
+    // ------ Custom render passes set up ------
+
+    const currentOptions = {
+        camera: cameraEntity.camera, // camera used to render those passes
+        samples: 0, // number of samples for multi-sampling
+        sceneColorMap: true,
+
+        // enable the pre-pass to generate the depth buffer, which is needed by the TAA
+        prepassEnabled: true,
+
+        // enable temporal anti-aliasing
+        taaEnabled: true
+    };
+
+    const setupRenderPass = () => {
+
+        // destroy existing pass if any
+        if (cameraEntity.camera.renderPasses.length > 0) {
+            cameraEntity.camera.renderPasses[0].destroy();
+        }
+
+        // Use a render pass camera frame, which is a render pass that implements typical rendering of a camera.
+        // Internally this sets up additional passes it needs, based on the options passed to it.
+        const renderPassCamera = new pcx.RenderPassCameraFrame(app, currentOptions);
+        renderPassCamera.bloomEnabled = false;
+
+        const composePass = renderPassCamera.composePass;
+        composePass.toneMapping = data.get('data.scene.tonemapping');
+        composePass.sharpness = currentOptions.taaEnabled ? 1 : 0;
+
+        // and set up these rendering passes to be used by the camera, instead of its default rendering
+        cameraEntity.camera.renderPasses = [renderPassCamera];
+
+        // the render passes render in HDR format, and so disable output tone mapping and gamma correction,
+        // as that is applied in the final compose pass
+        app.scene.toneMapping = pc.TONEMAP_LINEAR;
+        app.scene.gammaCorrection = pc.GAMMA_NONE;
+
+        // jitter the camera when TAA is enabled
+        cameraEntity.camera.jitter = currentOptions.taaEnabled ? 1 : 0;
+    };
+
+    setupRenderPass();
+
+    // ------
+
     // handle UI changes
     data.on('*:set', (/** @type {string} */ path, value) => {
         const propertyName = path.split('.')[1];
@@ -169,12 +220,22 @@ assetListLoader.load(() => {
                 // turn on / off depth write depending on the dithering of the color
                 material.depthWrite = value !== pc.DITHER_NONE;
             }
+
             material.update();
+
+            // if TAA property changes, we need to set up render passes again
+            if (propertyName === 'taa') {
+                if (currentOptions.taaEnabled !== value) {
+                    currentOptions.taaEnabled = value;
+                    setupRenderPass();
+                }
+            }
         });
     });
 
     // initial values
     data.set('data', {
+        taa: false,
         opacity: 0.5,
         opacityDither: pc.DITHER_BAYER8,
         opacityShadowDither: pc.DITHER_BAYER8

src/scene/materials/lit-material-options-builder.js

@@ -1,4 +1,9 @@
-import { CUBEPROJ_NONE, GAMMA_SRGBHDR, GAMMA_NONE, LIGHTTYPE_DIRECTIONAL, LIGHTTYPE_OMNI, LIGHTTYPE_SPOT, MASK_AFFECT_DYNAMIC, SHADER_FORWARDHDR, TONEMAP_LINEAR, SHADERDEF_INSTANCING, SHADERDEF_MORPH_NORMAL, SHADERDEF_MORPH_POSITION, SHADERDEF_MORPH_TEXTURE_BASED, SHADERDEF_SCREENSPACE, SHADERDEF_SKIN, SHADERDEF_NOSHADOW, SHADERDEF_TANGENTS, SPECULAR_BLINN, SPRITE_RENDERMODE_SIMPLE } from "../constants.js";
+import {
+    CUBEPROJ_NONE, GAMMA_SRGBHDR, GAMMA_NONE, LIGHTTYPE_DIRECTIONAL, LIGHTTYPE_OMNI, LIGHTTYPE_SPOT,
+    MASK_AFFECT_DYNAMIC, SHADER_FORWARDHDR, TONEMAP_LINEAR, SHADERDEF_INSTANCING, SHADERDEF_MORPH_NORMAL,
+    SHADERDEF_MORPH_POSITION, SHADERDEF_MORPH_TEXTURE_BASED, SHADERDEF_SCREENSPACE, SHADERDEF_SKIN,
+    SHADERDEF_NOSHADOW, SHADERDEF_TANGENTS, SPECULAR_BLINN, SPRITE_RENDERMODE_SIMPLE
+} from "../constants.js";
 
 class LitMaterialOptionsBuilder {
     static update(litOptions, material, scene, objDefs, pass, sortedLights) {
@@ -64,7 +69,6 @@ class LitMaterialOptionsBuilder {
         litOptions.alphaToCoverage = material.alphaToCoverage;
         litOptions.opacityFadesSpecular = material.opacityFadesSpecular;
         litOptions.opacityDither = material.opacityDither;
-        litOptions.opacityShadowDither = material.opacityShadowDither;
 
         litOptions.cubeMapProjection = CUBEPROJ_NONE;
 

src/scene/materials/standard-material-options-builder.js

@@ -7,7 +7,7 @@ import {
     GAMMA_NONE, GAMMA_SRGBHDR,
     LIGHTTYPE_DIRECTIONAL, LIGHTTYPE_OMNI, LIGHTTYPE_SPOT,
     MASK_AFFECT_DYNAMIC,
-    SHADER_FORWARDHDR,
+    SHADER_FORWARDHDR, SHADER_PREPASS_VELOCITY,
     SHADERDEF_DIRLM, SHADERDEF_INSTANCING, SHADERDEF_LM, SHADERDEF_MORPH_POSITION, SHADERDEF_MORPH_NORMAL, SHADERDEF_NOSHADOW, SHADERDEF_MORPH_TEXTURE_BASED,
     SHADERDEF_SCREENSPACE, SHADERDEF_SKIN, SHADERDEF_TANGENTS, SHADERDEF_UV0, SHADERDEF_UV1, SHADERDEF_VCOLOR, SHADERDEF_LMAMBIENT,
     TONEMAP_LINEAR,
@@ -45,7 +45,7 @@ class StandardMaterialOptionsBuilder {
     // Minimal options for Depth and Shadow passes
     updateMinRef(options, scene, stdMat, objDefs, pass, sortedLights) {
         this._updateSharedOptions(options, scene, stdMat, objDefs, pass);
-        this._updateMinOptions(options, stdMat);
+        this._updateMinOptions(options, stdMat, pass);
         this._updateUVOptions(options, stdMat, objDefs, true);
     }
 
@@ -188,9 +188,13 @@ class StandardMaterialOptionsBuilder {
         }
     }
 
-    _updateMinOptions(options, stdMat) {
+    _updateMinOptions(options, stdMat, pass) {
         options.opacityTint = stdMat.opacity !== 1 && (stdMat.blendType !== BLEND_NONE || stdMat.opacityShadowDither !== DITHER_NONE);
-        options.litOptions.opacityShadowDither = stdMat.opacityShadowDither;
+
+        // pre-pass uses the same dither setting as forward pass, otherwise shadow dither
+        const isPrepass = pass === SHADER_PREPASS_VELOCITY;
+        options.litOptions.opacityShadowDither = isPrepass ? stdMat.opacityDither : stdMat.opacityShadowDither;
+
         options.litOptions.lights = [];
     }
 

src/scene/renderer/renderer.js

@@ -50,7 +50,6 @@ const tempSphere = new BoundingSphere();
 const _flipYMat = new Mat4().setScale(1, -1, 1);
 const _tempLightSet = new Set();
 const _tempLayerSet = new Set();
-const _tempVec4 = new Vec4();
 
 // Converts a projection matrix in OpenGL style (depth range of -1..1) to a DirectX style (depth range of 0..1).
 const _fixProjRangeMat = new Mat4().set([
@@ -221,6 +220,9 @@ class Renderer {
         this.cameraParamsId = scope.resolve('camera_params');
         this.viewIndexId = scope.resolve('view_index');
 
+        this.blueNoiseJitterVersion = 0;
+        this.blueNoiseJitterVec = new Vec4();
+        this.blueNoiseJitterData = new Float32Array(4);
         this.blueNoiseJitterId = scope.resolve('blueNoiseJitter');
         this.blueNoiseTextureId = scope.resolve('blueNoiseTex32');
 
@@ -375,7 +377,6 @@ class Renderer {
 
             // camera jitter
             const { jitter } = camera;
-            let noise = Vec4.ZERO;
             let jitterX = 0;
             let jitterY = 0;
             if (jitter > 0) {
@@ -399,11 +400,19 @@ class Renderer {
                 projMatSkybox.data[8] = jitterX;
                 projMatSkybox.data[9] = jitterY;
 
-                // blue noise vec4 - only set when jitter is enabled
-                noise = this.blueNoise.vec4(_tempVec4);
+                // blue noise vec4 - only use when jitter is enabled
+                if (this.blueNoiseJitterVersion !== this.device.renderVersion) {
+                    this.blueNoiseJitterVersion = this.device.renderVersion;
+                    this.blueNoise.vec4(this.blueNoiseJitterVec);
+                }
             }
 
-            this.blueNoiseJitterId.setValue([noise.x, noise.y, noise.z, noise.w]);
+            const jitterVec = jitter > 0 ? this.blueNoiseJitterVec : Vec4.ZERO;
+            this.blueNoiseJitterData[0] = jitterVec.x;
+            this.blueNoiseJitterData[1] = jitterVec.y;
+            this.blueNoiseJitterData[2] = jitterVec.z;
+            this.blueNoiseJitterData[3] = jitterVec.w;
+            this.blueNoiseJitterId.setValue(this.blueNoiseJitterData);
 
             this.projId.setValue(projMat.data);
             this.projSkyboxId.setValue(projMatSkybox.data);

src/scene/shader-lib/programs/lit-shader.js

@@ -212,7 +212,7 @@ class LitShader {
 
         codeBody += "   vPositionW    = getWorldPosition();\n";
 
-        if (this.options.pass === SHADER_DEPTH) {
+        if (this.options.pass === SHADER_DEPTH || this.options.pass === SHADER_PREPASS_VELOCITY) {
             code += 'varying float vDepth;\n';
             code += '#ifndef VIEWMATRIX\n';
             code += '#define VIEWMATRIX\n';
@@ -454,14 +454,9 @@ class LitShader {
     }
 
     _fsGetPrePassVelocityCode() {
-        const code = `
-            void main(void)
-            {
-                gl_FragColor = vec4(1, 0, 0, 1);
-            }
-            `;
 
-        return code;
+        // till the velocity is implemented, just output the depth
+        return this._fsGetDepthPassCode();
     }
 
     _fsGetShadowPassCode() {