Use stencil test instead of tile mask approach

Remove tile masks for raster. Rendering happens in Z-descending order.
All the tiles with the same Z use the same stencil value. Parent uses lower stencil value so that the area covered by children gets masked.
Stencil ref values continue range used in _tileClippingMaskIDs.

src/render/draw_hillshade.js

@@ -23,20 +23,23 @@ function drawHillshade(painter: Painter, sourceCache: SourceCache, layer: Hillsh
     const sourceMaxZoom = sourceCache.getSource().maxzoom;
 
     const depthMode = painter.depthModeForSublayer(0, DepthMode.ReadOnly);
-    const stencilMode = StencilMode.disabled;
     const colorMode = painter.colorModeForRenderPass();
 
-    for (const tileID of tileIDs) {
-        const tile = sourceCache.getTile(tileID);
+    const [getStencilMode, done, coords] = painter.renderPass === 'translucent' ?
+        painter.setupStencilingForOverdraw(tileIDs) : [(_) => StencilMode.disabled, () => {}, tileIDs];
+
+    for (const coord of coords) {
+        const tile = sourceCache.getTile(coord);
         if (tile.needsHillshadePrepare && painter.renderPass === 'offscreen') {
-            prepareHillshade(painter, tile, layer, sourceMaxZoom, depthMode, stencilMode, colorMode);
+            prepareHillshade(painter, tile, layer, sourceMaxZoom, depthMode, StencilMode.disabled, colorMode);
             continue;
         } else if (painter.renderPass === 'translucent') {
-            renderHillshade(painter, tile, layer, depthMode, stencilMode, colorMode);
+            renderHillshade(painter, tile, layer, depthMode, getStencilMode(coord), colorMode);
         }
     }
 
     context.viewport.set([0, 0, painter.width, painter.height]);
+    done();
 }
 
 function renderHillshade(painter, tile, layer, depthMode, stencilMode, colorMode) {
@@ -52,15 +55,9 @@ function renderHillshade(painter, tile, layer, depthMode, stencilMode, colorMode
 
     const uniformValues = hillshadeUniformValues(painter, tile, layer);
 
-    if (tile.maskedBoundsBuffer && tile.maskedIndexBuffer && tile.segments) {
-        program.draw(context, gl.TRIANGLES, depthMode, stencilMode, colorMode, CullFaceMode.disabled,
-            uniformValues, layer.id, tile.maskedBoundsBuffer,
-            tile.maskedIndexBuffer, tile.segments);
-    } else {
-        program.draw(context, gl.TRIANGLES, depthMode, stencilMode, colorMode, CullFaceMode.disabled,
-            uniformValues, layer.id, painter.rasterBoundsBuffer,
-            painter.quadTriangleIndexBuffer, painter.rasterBoundsSegments);
-    }
+    program.draw(context, gl.TRIANGLES, depthMode, stencilMode, colorMode, CullFaceMode.disabled,
+        uniformValues, layer.id, painter.rasterBoundsBuffer,
+        painter.quadTriangleIndexBuffer, painter.rasterBoundsSegments);
 }
 
 // hillshade rendering is done in two steps. the prepare step first calculates the slope of the terrain in the x and y

src/render/draw_raster.js

@@ -16,18 +16,24 @@ import type {OverscaledTileID} from '../source/tile_id';
 
 export default drawRaster;
 
-function drawRaster(painter: Painter, sourceCache: SourceCache, layer: RasterStyleLayer, coords: Array<OverscaledTileID>) {
+function drawRaster(painter: Painter, sourceCache: SourceCache, layer: RasterStyleLayer, tileIDs: Array<OverscaledTileID>) {
     if (painter.renderPass !== 'translucent') return;
     if (layer.paint.get('raster-opacity') === 0) return;
+    if (!tileIDs.length) return;
 
     const context = painter.context;
     const gl = context.gl;
     const source = sourceCache.getSource();
     const program = painter.useProgram('raster');
 
-    const stencilMode = StencilMode.disabled;
     const colorMode = painter.colorModeForRenderPass();
-    const minTileZ = coords.length && coords[0].overscaledZ;
+
+    const [getStencilMode, done, coords] = source instanceof ImageSource ?
+        [(_) => StencilMode.disabled, () => {}, tileIDs] :
+        painter.setupStencilingForOverdraw(tileIDs);
+
+    const minTileZ = coords[coords.length - 1].overscaledZ;
+
     const align = !painter.options.moving;
     for (const coord of coords) {
         // Set the lower zoom level to sublayer 0, and higher zoom levels to higher sublayers
@@ -64,20 +70,16 @@ function drawRaster(painter: Painter, sourceCache: SourceCache, layer: RasterSty
         const uniformValues = rasterUniformValues(posMatrix, parentTL || [0, 0], parentScaleBy || 1, fade, layer);
 
         if (source instanceof ImageSource) {
-            program.draw(context, gl.TRIANGLES, depthMode, stencilMode, colorMode, CullFaceMode.disabled,
+            program.draw(context, gl.TRIANGLES, depthMode, StencilMode.disabled, colorMode, CullFaceMode.disabled,
                 uniformValues, layer.id, source.boundsBuffer,
                 painter.quadTriangleIndexBuffer, source.boundsSegments);
-        } else if (tile.maskedBoundsBuffer && tile.maskedIndexBuffer && tile.segments) {
-            program.draw(context, gl.TRIANGLES, depthMode, stencilMode, colorMode, CullFaceMode.disabled,
-                uniformValues, layer.id, tile.maskedBoundsBuffer,
-                tile.maskedIndexBuffer, tile.segments, layer.paint,
-                painter.transform.zoom);
         } else {
-            program.draw(context, gl.TRIANGLES, depthMode, stencilMode, colorMode, CullFaceMode.disabled,
+            program.draw(context, gl.TRIANGLES, depthMode, getStencilMode(coord), colorMode, CullFaceMode.disabled,
                 uniformValues, layer.id, painter.rasterBoundsBuffer,
                 painter.quadTriangleIndexBuffer, painter.rasterBoundsSegments);
         }
     }
+    done();
 }
 
 function getFadeValues(tile, parentTile, sourceCache, layer, transform) {

src/render/painter.js

@@ -22,7 +22,6 @@ import StencilMode from '../gl/stencil_mode';
 import ColorMode from '../gl/color_mode';
 import CullFaceMode from '../gl/cull_face_mode';
 import Texture from './texture';
-import updateTileMasks from './tile_mask';
 import {clippingMaskUniformValues} from './program/clipping_mask_program';
 import Color from '../style-spec/util/color';
 import symbol from './draw_symbol';
@@ -296,6 +295,35 @@ class Painter {
         return new StencilMode({func: gl.EQUAL, mask: 0xFF}, this._tileClippingMaskIDs[tileID.key], 0x00, gl.KEEP, gl.KEEP, gl.REPLACE);
     }
 
+    /*
+     * Sort coordinates by Z as drawing tiles is done in Z-descending order.
+     * All children with the same Z write the same stencil value.  Children
+     * stencil values are greater than parent's.  Stencil ref values continue
+     * range used in _tileClippingMaskIDs.
+     *
+     * Returns [getStencilMode(coordinate), cleanup(), sortedCoords].
+     */
+    setupStencilingForOverdraw(tileIDs: Array<OverscaledTileID>): [((coord: OverscaledTileID) => StencilMode), (() => void), Array<OverscaledTileID>] {
+        this.currentStencilSource = undefined;
+        const gl = this.context.gl;
+        const coords = tileIDs.sort((a, b) => b.overscaledZ - a.overscaledZ);
+        const minTileZ = coords[coords.length - 1].overscaledZ;
+        const stencilValues = coords[0].overscaledZ - minTileZ + 1;
+        if (this.nextStencilID + stencilValues > 256) {
+            this.clearStencil();
+        }
+        const stencilMode = new StencilMode({func: gl.GEQUAL, mask: 0xFF}, 0, 0xFF, gl.KEEP, gl.KEEP, gl.REPLACE);
+        const painter = this;
+        const getStencilMode = (coord) => {
+            stencilMode.ref = painter.nextStencilID + coord.overscaledZ - minTileZ;
+            return stencilMode;
+        };
+        const done = () => {
+            painter.nextStencilID += stencilValues;
+        };
+        return [getStencilMode, done, coords];
+    }
+
     colorModeForRenderPass(): $ReadOnly<ColorMode> {
         const gl = this.context.gl;
         if (this._showOverdrawInspector) {
@@ -360,15 +388,6 @@ class Painter {
             coordsDescendingSymbol[id] = sourceCache.getVisibleCoordinates(true).reverse();
         }
 
-        for (const id in sourceCaches) {
-            const sourceCache = sourceCaches[id];
-            const source = sourceCache.getSource();
-            if (source.type !== 'raster' && source.type !== 'raster-dem') continue;
-            const visibleTiles = [];
-            for (const coord of coordsAscending[id]) visibleTiles.push(sourceCache.getTile(coord));
-            updateTileMasks(visibleTiles, this.context);
-        }
-
         this.opaquePassCutoff = Infinity;
         for (let i = 0; i < layerIds.length; i++) {
             const layerId = layerIds[i];

src/source/geojson_source.js

@@ -326,7 +326,6 @@ class GeoJSONSource extends Evented implements Source {
 
     unloadTile(tile: Tile) {
         tile.unloadVectorData();
-        tile.clearMask();
         this.actor.send('removeTile', {uid: tile.uid, type: this.type, source: this.id});
     }
 

src/source/raster_dem_tile_source.js

@@ -123,7 +123,6 @@ class RasterDEMTileSource extends RasterTileSource implements Source {
         }
         if (tile.dem) delete tile.dem;
         delete tile.neighboringTiles;
-        tile.clearMask();
 
         tile.state = 'unloaded';
         if (tile.actor) {

src/source/raster_tile_source.js

@@ -158,7 +158,6 @@ class RasterTileSource extends Evented implements Source {
 
     unloadTile(tile: Tile, callback: Callback<void>) {
         if (tile.texture) this.map.painter.saveTileTexture(tile.texture);
-        tile.clearMask();
         callback();
     }