Support transparent shadow

e2e/case/shadow-transparent.ts

@@ -0,0 +1,61 @@
+/**
+ * @title Shadow Transparent
+ * @category Shadow
+ */
+
+import {
+  Camera,
+  DirectLight,
+  GLTFResource,
+  Logger,
+  MeshRenderer,
+  PBRMaterial,
+  PrimitiveMesh,
+  ShadowResolution,
+  ShadowType,
+  Vector3,
+  WebGLEngine,
+  WebGLMode
+} from "@galacean/engine";
+import { initScreenshot, updateForE2E } from "./.mockForE2E";
+Logger.enable();
+
+WebGLEngine.create({
+  canvas: "canvas",
+  graphicDeviceOptions: {
+    webGLMode: WebGLMode.WebGL2
+  }
+}).then((engine) => {
+  engine.canvas.resizeByClientSize();
+  const scene = engine.sceneManager.activeScene;
+  const rootEntity = scene.createRootEntity();
+  scene.shadowResolution = ShadowResolution.Medium;
+  scene.shadowDistance = 10;
+  const cameraEntity = rootEntity.createChild("camera_node");
+  cameraEntity.transform.setPosition(0, 2, 3);
+  cameraEntity.transform.lookAt(new Vector3(0));
+  const camera = cameraEntity.addComponent(Camera);
+  const lightEntity = rootEntity.createChild("light_node");
+  const light = lightEntity.addComponent(DirectLight);
+  lightEntity.transform.setPosition(-6, 10, 0);
+  lightEntity.transform.lookAt(new Vector3(0, 0, -10));
+  light.shadowType = ShadowType.Hard;
+
+  const planeEntity = rootEntity.createChild("plane_node");
+  const renderer = planeEntity.addComponent(MeshRenderer);
+  renderer.mesh = PrimitiveMesh.createPlane(engine, 10, 10);
+  const planeMaterial = new PBRMaterial(engine);
+  renderer.setMaterial(planeMaterial);
+
+  engine.resourceManager
+    .load<GLTFResource>("https://mdn.alipayobjects.com/oasis_be/afts/file/A*kgYIRo36270AAAAAAAAAAAAADkp5AQ/bottle.glb")
+    .then((asset) => {
+      const defaultSceneRoot = asset.instantiateSceneRoot();
+      rootEntity.addChild(defaultSceneRoot);
+      defaultSceneRoot.transform.scale.set(0.05, 0.05, 0.05);
+      scene.enableTransparentShadow = true;
+
+      updateForE2E(engine, 500);
+      initScreenshot(engine, camera);
+    });
+});

e2e/config.ts

@@ -111,6 +111,11 @@ export const E2E_CONFIG = {
       category: "Shadow",
       caseFileName: "shadow-basic",
       threshold: 0.2
+    },
+    transparent: {
+      category: "Shadow",
+      caseFileName: "shadow-transparent",
+      threshold: 0.2
     }
   },
   Primitive: {

packages/core/src/RenderPipeline/BasicRenderPipeline.ts

@@ -237,7 +237,17 @@ export class BasicRenderPipeline {
 
     let renderQueueAddedFlags = RenderQueueAddedFlag.None;
     for (let i = 0, n = shaderPasses.length; i < n; i++) {
-      const renderQueueType = (shaderPasses[i]._renderState ?? renderStates[i]).renderQueueType;
+      // Get render queue type
+      let renderQueueType: RenderQueueType;
+      const shaderPass = shaderPasses[i];
+      const renderState = shaderPass._renderState;
+      if (renderState) {
+        renderState._applyRenderQueueByShaderData(shaderPass._renderStateDataMap, element.material.shaderData);
+        renderQueueType = renderState.renderQueueType;
+      } else {
+        renderQueueType = renderStates[i].renderQueueType;
+      }
+
       if (renderQueueAddedFlags & (<RenderQueueAddedFlag>(1 << renderQueueType))) {
         continue;
       }
@@ -280,7 +290,12 @@ export class BasicRenderPipeline {
     program.uploadAll(program.materialUniformBlock, material.shaderData);
     program.uploadUnGroupTextures();
 
-    (pass._renderState || material.renderState)._apply(engine, false, pass._renderStateDataMap, material.shaderData);
+    (pass._renderState || material.renderState)._applyStates(
+      engine,
+      false,
+      pass._renderStateDataMap,
+      material.shaderData
+    );
     rhi.drawPrimitive(mesh._primitive, mesh.subMesh, program);
   }
 

packages/core/src/RenderPipeline/PipelineUtils.ts

@@ -147,7 +147,7 @@ export class PipelineUtils {
     program.uploadAll(program.materialUniformBlock, blitMaterial.shaderData);
     program.uploadUnGroupTextures();
 
-    (pass._renderState || blitMaterial.renderState)._apply(
+    (pass._renderState || blitMaterial.renderState)._applyStates(
       engine,
       false,
       pass._renderStateDataMap,

packages/core/src/RenderPipeline/RenderQueue.ts

@@ -179,7 +179,7 @@ export class RenderQueue {
           }
 
           const renderState = shaderPass._renderState ?? renderStates[j];
-          renderState._apply(
+          renderState._applyStates(
             engine,
             renderer.entity.transform._isFrontFaceInvert(),
             shaderPass._renderStateDataMap,

packages/core/src/RenderPipeline/SpriteBatcher.ts

@@ -125,7 +125,12 @@ export class SpriteBatcher extends Basic2DBatcher {
       program.uploadAll(program.rendererUniformBlock, renderer.shaderData);
       program.uploadAll(program.materialUniformBlock, material.shaderData);
 
-      (pass._renderState || material.renderState)._apply(engine, false, pass._renderStateDataMap, material.shaderData);
+      (pass._renderState || material.renderState)._applyStates(
+        engine,
+        false,
+        pass._renderStateDataMap,
+        material.shaderData
+      );
       engine._hardwareRenderer.drawPrimitive(primitive, subMesh, program);
 
       maskManager.postRender(renderer);

packages/core/src/RenderPipeline/SpriteMaskBatcher.ts

@@ -97,7 +97,12 @@ export class SpriteMaskBatcher extends Basic2DBatcher {
       program.uploadAll(program.rendererUniformBlock, renderer.shaderData);
       program.uploadAll(program.materialUniformBlock, material.shaderData);
 
-      (pass._renderState || material.renderState)._apply(engine, false, pass._renderStateDataMap, material.shaderData);
+      (pass._renderState || material.renderState)._applyStates(
+        engine,
+        false,
+        pass._renderStateDataMap,
+        material.shaderData
+      );
 
       engine._hardwareRenderer.drawPrimitive(primitive, subMesh, program);
     }

packages/core/src/Scene.ts

@@ -73,6 +73,7 @@ export class Scene extends EngineObject {
   private _fogParams: Vector4 = new Vector4();
   private _isActive: boolean = true;
   private _sun: DirectLight | null;
+  private _enableTransparentShadow = false;
 
   /**
    * Whether the scene is active.
@@ -241,6 +242,24 @@ export class Scene extends EngineObject {
     this._sun = light;
   }
 
+  /**
+   * Whether to enable transparent shadow.
+   */
+  get enableTransparentShadow(): boolean {
+    return this._enableTransparentShadow;
+  }
+
+  set enableTransparentShadow(value: boolean) {
+    if (value !== this._enableTransparentShadow) {
+      this._enableTransparentShadow = value;
+      if (value) {
+        this.shaderData.enableMacro("SCENE_ENABLE_TRANSPARENT_SHADOW");
+      } else {
+        this.shaderData.disableMacro("SCENE_ENABLE_TRANSPARENT_SHADOW");
+      }
+    }
+  }
+
   /**
    * Create scene.
    * @param engine - Engine

packages/core/src/material/BaseMaterial.ts

@@ -8,6 +8,9 @@ import { RenderFace } from "./enums/RenderFace";
 import { Material } from "./Material";
 
 export class BaseMaterial extends Material {
+  /** @internal */
+  static _shadowCasterRenderQueueProp = ShaderProperty.getByName("material_ShadowCasterRenderQueue");
+
   protected static _baseTextureMacro: ShaderMacro = ShaderMacro.getByName("MATERIAL_HAS_BASETEXTURE");
   protected static _normalTextureMacro: ShaderMacro = ShaderMacro.getByName("MATERIAL_HAS_NORMALTEXTURE");
   protected static _emissiveTextureMacro: ShaderMacro = ShaderMacro.getByName("MATERIAL_HAS_EMISSIVETEXTURE");
@@ -63,7 +66,7 @@ export class BaseMaterial extends Material {
   }
 
   /**
-   * Whethor transparent of first shader pass render state.
+   * Whether transparent of first shader pass render state.
    */
   get isTransparent(): boolean {
     return this._isTransparent;
@@ -72,6 +75,20 @@ export class BaseMaterial extends Material {
   set isTransparent(value: boolean) {
     if (value !== this._isTransparent) {
       this.setIsTransparent(0, value);
+
+      const { shaderData } = this;
+      if (value) {
+        // Use alpha test queue to simulate transparent shadow
+        shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
+      } else {
+        const alphaCutoff = shaderData.getFloat(BaseMaterial._alphaCutoffProp);
+        if (alphaCutoff) {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
+        } else {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.Opaque);
+        }
+      }
+
       this._isTransparent = value;
     }
   }
@@ -106,8 +123,14 @@ export class BaseMaterial extends Material {
     if (shaderData.getFloat(BaseMaterial._alphaCutoffProp) !== value) {
       if (value) {
         shaderData.enableMacro(BaseMaterial._alphaCutoffMacro);
+        shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
       } else {
         shaderData.disableMacro(BaseMaterial._alphaCutoffMacro);
+        if (this._isTransparent) {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.AlphaTest);
+        } else {
+          shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.Opaque);
+        }
       }
 
       const { renderStates } = this;
@@ -123,7 +146,6 @@ export class BaseMaterial extends Material {
             : RenderQueueType.Opaque;
         }
       }
-
       shaderData.setFloat(BaseMaterial._alphaCutoffProp, value);
     }
   }
@@ -149,7 +171,10 @@ export class BaseMaterial extends Material {
    */
   constructor(engine: Engine, shader: Shader) {
     super(engine, shader);
-    this.shaderData.setFloat(BaseMaterial._alphaCutoffProp, 0);
+
+    const { shaderData } = this;
+    shaderData.setFloat(BaseMaterial._alphaCutoffProp, 0);
+    shaderData.setFloat(BaseMaterial._shadowCasterRenderQueueProp, RenderQueueType.Opaque);
   }
 
   /**
@@ -163,19 +188,21 @@ export class BaseMaterial extends Material {
       throw "Pass should less than pass count.";
     }
     const renderState = renderStates[passIndex];
+    const { shaderData } = this;
 
     if (isTransparent) {
       renderState.blendState.targetBlendState.enabled = true;
       renderState.depthState.writeEnabled = false;
       renderState.renderQueueType = RenderQueueType.Transparent;
-      this.shaderData.enableMacro(BaseMaterial._transparentMacro);
+      shaderData.enableMacro(BaseMaterial._transparentMacro);
     } else {
       renderState.blendState.targetBlendState.enabled = false;
       renderState.depthState.writeEnabled = true;
-      renderState.renderQueueType = this.shaderData.getFloat(BaseMaterial._alphaCutoffProp)
+
+      renderState.renderQueueType = shaderData.getFloat(BaseMaterial._alphaCutoffProp)
         ? RenderQueueType.AlphaTest
         : RenderQueueType.Opaque;
-      this.shaderData.disableMacro(BaseMaterial._transparentMacro);
+      shaderData.disableMacro(BaseMaterial._transparentMacro);
     }
   }
 

packages/core/src/shader/ShaderPool.ts

@@ -1,4 +1,5 @@
 import { PipelineStage } from "../RenderPipeline/enums/PipelineStage";
+import { BaseMaterial } from "../material/BaseMaterial";
 import blitFs from "../shaderlib/extra/Blit.fs.glsl";
 import blitVs from "../shaderlib/extra/Blit.vs.glsl";
 import skyProceduralFs from "../shaderlib/extra/SkyProcedural.fs.glsl";
@@ -26,6 +27,8 @@ import unlitFs from "../shaderlib/extra/unlit.fs.glsl";
 import unlitVs from "../shaderlib/extra/unlit.vs.glsl";
 import { Shader } from "./Shader";
 import { ShaderPass } from "./ShaderPass";
+import { RenderStateElementKey } from "./enums/RenderStateElementKey";
+import { RenderState } from "./state";
 
 /**
  * Internal shader pool.
@@ -36,6 +39,10 @@ export class ShaderPool {
     const shadowCasterPass = new ShaderPass("ShadowCaster", shadowMapVs, shadowMapFs, {
       pipelineStage: PipelineStage.ShadowCaster
     });
+    shadowCasterPass._renderState = new RenderState();
+    shadowCasterPass._renderStateDataMap[RenderStateElementKey.RenderQueueType] =
+      BaseMaterial._shadowCasterRenderQueueProp;
+
     const depthOnlyPass = new ShaderPass("DepthOnly", depthOnlyVs, depthOnlyFs, {
       pipelineStage: PipelineStage.DepthOnly
     });

packages/core/src/shader/state/RenderState.ts

@@ -30,30 +30,15 @@ export class RenderState {
 
   /**
    * @internal
-   * @todo Should merge when we can delete material render state.
    */
-  _applyShaderDataValue(renderStateDataMap: Record<number, ShaderProperty>, shaderData: ShaderData): void {
-    this.blendState._applyShaderDataValue(renderStateDataMap, shaderData);
-    this.depthState._applyShaderDataValue(renderStateDataMap, shaderData);
-    this.stencilState._applyShaderDataValue(renderStateDataMap, shaderData);
-    this.rasterState._applyShaderDataValue(renderStateDataMap, shaderData);
-
-    const renderQueueType = renderStateDataMap[RenderStateElementKey.RenderQueueType];
-    if (renderQueueType !== undefined) {
-      this.renderQueueType = shaderData.getFloat(renderQueueType) ?? RenderQueueType.Opaque;
-    }
-  }
-
-  /**
-   * @internal
-   */
-  _apply(
+  _applyStates(
     engine: Engine,
     frontFaceInvert: boolean,
     renderStateDataMap: Record<number, ShaderProperty>,
     shaderData: ShaderData
   ): void {
-    renderStateDataMap && this._applyShaderDataValue(renderStateDataMap, shaderData);
+    // @todo: Should merge when we can delete material render state
+    renderStateDataMap && this._applyStatesByShaderData(renderStateDataMap, shaderData);
     const hardwareRenderer = engine._hardwareRenderer;
     const lastRenderState = engine._lastRenderState;
     const context = engine._renderContext;
@@ -66,4 +51,22 @@ export class RenderState {
       context.flipProjection ? !frontFaceInvert : frontFaceInvert
     );
   }
+
+  /**
+   * @internal
+   * @todo Should merge when we can delete material render state
+   */
+  _applyRenderQueueByShaderData(renderStateDataMap: Record<number, ShaderProperty>, shaderData: ShaderData): void {
+    const renderQueueType = renderStateDataMap[RenderStateElementKey.RenderQueueType];
+    if (renderQueueType !== undefined) {
+      this.renderQueueType = shaderData.getFloat(renderQueueType) ?? RenderQueueType.Opaque;
+    }
+  }
+
+  private _applyStatesByShaderData(renderStateDataMap: Record<number, ShaderProperty>, shaderData: ShaderData): void {
+    this.blendState._applyShaderDataValue(renderStateDataMap, shaderData);
+    this.depthState._applyShaderDataValue(renderStateDataMap, shaderData);
+    this.stencilState._applyShaderDataValue(renderStateDataMap, shaderData);
+    this.rasterState._applyShaderDataValue(renderStateDataMap, shaderData);
+  }
 }