[MAPS3D-1411] Remove remaining shadows after fill-extrusion cutoff (i…

…nternal-1640)

* Add shadow fade-out based on cutoff opacity

* Add fill-extrusion-cutoff-fade-range render test

* Fix fill-extrusion-cutoff-fade-range working differently in gl-js and gl-native

* Update render test

* Update render test expectation

* Increase render test tolerance

3d-style/shaders/_prelude_shadow.fragment.glsl

@@ -116,6 +116,15 @@ float shadowed_light_factor_normal(vec3 N, highp vec4 light_view_pos0, highp vec
     return mix(0.0, (1.0 - (u_shadow_intensity * occlusion)) * NDotL, step(0.0, NDotL));
 }
 
+float shadowed_light_factor_normal_opacity(vec3 N, highp vec4 light_view_pos0, highp vec4 light_view_pos1, float view_depth, float shadow_opacity) {
+    float NDotL = dot(N, u_shadow_direction);
+
+    float bias = calculate_shadow_bias(NDotL);
+    float occlusion = shadow_occlusion(light_view_pos0, light_view_pos1, view_depth, bias) * shadow_opacity;
+
+    return mix(0.0, (1.0 - (u_shadow_intensity * occlusion)) * NDotL, step(0.0, NDotL));
+}
+
 float shadowed_light_factor_normal_unbiased(vec3 N, highp vec4 light_view_pos0, highp vec4 light_view_pos1, float view_depth) {
     float NDotL = dot(N, u_shadow_direction);
 

src/shaders/fill_extrusion.fragment.glsl

@@ -102,7 +102,15 @@ float flood_radiance = 0.0;
 
     color.rgb = mix(litColor, floodLitColor, flood_radiance);
 #else // FLOOD_LIGHT
-    float shadowed_lighting_factor = shadowed_light_factor_normal(normal, v_pos_light_view_0, v_pos_light_view_1, 1.0 / gl_FragCoord.w);
+    float shadowed_lighting_factor;
+#ifdef RENDER_CUTOFF
+    shadowed_lighting_factor = shadowed_light_factor_normal_opacity(normal, v_pos_light_view_0, v_pos_light_view_1, 1.0 / gl_FragCoord.w, v_cutoff_opacity);
+    if (v_cutoff_opacity == 0.0) {
+        discard;
+    }
+#else // RENDER_CUTOFF
+    shadowed_lighting_factor = shadowed_light_factor_normal(normal, v_pos_light_view_0, v_pos_light_view_1, 1.0 / gl_FragCoord.w);
+#endif // RENDER_CUTOFF
     color.rgb = apply_lighting(color.rgb, normal, shadowed_lighting_factor);
 #endif // !FLOOD_LIGHT 
 #else // RENDER_SHADOWS

src/shaders/fill_extrusion.vertex.glsl

@@ -134,7 +134,7 @@ void main() {
     vec3 centroid_random = vec3(centroid_pos.xy, centroid_pos.x + centroid_pos.y + 1.0);
     vec3 ground_pos = centroid_pos.x == 0.0 ? pos.xyz : (centroid_random / 8.0);
     vec4 ground = u_matrix * vec4(ground_pos.xy, ele, 1.0);
-    cutoff = max(0.01, cutoff_opacity(u_cutoff_params, ground.z));
+    cutoff = cutoff_opacity(u_cutoff_params, ground.z);
     if (centroid_pos.y != 0.0 && centroid_pos.x != 0.0) {
         vec3 g = floor(ground_pos);
         vec3 mod_ = centroid_random - g * 8.0;
@@ -144,10 +144,11 @@ void main() {
         }
     }
     float cutoff_scale = cutoff;
+    v_cutoff_opacity = cutoff;
 
     scaled_pos.z = mix(c_ele, h, cutoff_scale);
 #endif
-    float hidden = float((centroid_pos.x == 0.0 && centroid_pos.y == 1.0) || (cutoff < 0.01 && centroid_pos.x != 0.0));
+    float hidden = float((centroid_pos.x == 0.0 && centroid_pos.y == 1.0) || (cutoff == 0.0 && centroid_pos.x != 0.0));
 
     gl_Position = mix(u_matrix * vec4(scaled_pos, 1), AWAY, hidden);
     h = h - ele;

test/integration/render-tests/fill-extrusion-cutoff-fade-range/style.json

@@ -0,0 +1,79 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "height": 512,
+      "width": 512,
+      "allowed": 0.0023
+    }
+  },
+  "lights": [
+    {
+      "type": "ambient",
+      "id": "environment",
+      "properties": {
+        "intensity": 0.15,
+        "color": "rgba(100, 140, 255, 1.0)"
+      }
+    },
+    {
+      "type": "directional",
+      "id": "sun_light",
+      "properties": {
+        "intensity": 0.9,
+        "cast-shadows": true,
+        "shadow-intensity": 1.0,
+        "direction": [300, 75],
+        "color": "rgba(255, 255, 100, 1.0)"
+      }
+    }
+  ],
+  "sources": {
+    "mapbox": {
+      "type": "vector",
+      "maxzoom": 16,
+      "tiles": [
+        "local://tiles/{z}-{x}-{y}.mvt"
+      ]
+    }
+  },
+  "pitch": 65,
+  "bearing": 90,
+  "zoom": 15.3,
+  "center": [
+    -122.449283,
+    37.749211
+  ],
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "lightgreen"
+      }
+    },
+    {
+      "id": "road",
+      "type": "line",
+      "source": "mapbox",
+      "source-layer": "road",
+      "paint": {
+          "line-color": "darkgray",
+          "line-width": 10
+      }
+    },
+    {
+      "id": "extrusion",
+      "type": "fill-extrusion",
+      "source": "mapbox",
+      "source-layer": "building",
+      "minzoom": 15,
+      "paint": {
+        "fill-extrusion-color": "white",
+        "fill-extrusion-opacity": 1.0,
+        "fill-extrusion-height": 50.0,
+        "fill-extrusion-cutoff-fade-range": 1.0
+      }
+    }
+  ]
+}