[MAPS3D-1061] Fix flickering and several errors in shadow tile cover (#…

…786)

* Fix flickering and several errors in shadow tile cover

Sorting of shadowCasterTiles was incorrect so we were not removing repeated tiles. We also were creating shadow tile covers for raster sources, which was incorrect and hitting memory/performance. Finally the overscaledZoom used for the tilecover was updated to match what we have in internal

* Test remove subdivs

* Fix childOf to behave the same as native and avoid subdiving shadow caster tiles

src/geo/transform.js

@@ -851,72 +851,46 @@ class Transform {
             }
         }
 
-        let finished = false;
-
-        while (!finished) {
-            const subdivIds = [];
-
-            // Remove duplicates from new ids
-            if (out.length > 1) {
-                out.sort((a, b) => {
-                    return a.overscaledZ - b.overscaledZ ||
+        // Remove duplicates from new ids
+        if (out.length > 1) {
+            out.sort((a, b) => {
+                return a.overscaledZ - b.overscaledZ ||
                         a.wrap - b.wrap ||
                         a.canonical.z - b.canonical.z ||
                         a.canonical.x - b.canonical.x ||
                         a.canonical.y - b.canonical.y;
-                });
-
-                let i = 0;
-                let j = 0;
-                while (j < out.length) {
-                    if (!out[j].equals(out[i])) {
-                        out[++i] = out[j++];
-                    } else {
-                        ++j;
-                    }
-                }
-                out.length = i + 1;
-            }
-
-            // Remove higher zoom new IDs that overlap with other new IDs
-            const nonOverlappingIds = [];
+            });
 
-            for (const id of out) {
-                if (!out.some(ancestorCandidate => id.isChildOf(ancestorCandidate))) {
-                    nonOverlappingIds.push(id);
+            let i = 0;
+            let j = 0;
+            while (j < out.length) {
+                if (!out[j].equals(out[i])) {
+                    out[++i] = out[j++];
+                } else {
+                    ++j;
                 }
             }
+            out.length = i + 1;
+        }
 
-            // Remove new IDs that overlap with old IDs
-            out = nonOverlappingIds.filter(newId => !coveringTiles.some(oldId => {
-                // Subdivide parents of existing IDs
-                if (newId.overscaledZ < maxZoom && oldId.isChildOf(newId)) {
-                    for (let x = 0; x < 2; ++x) {
-                        for (let y = 0; y < 2; ++y) {
-                            const id = new OverscaledTileID(newId.overscaledZ + 1,
-                                                            newId.wrap,
-                                                            newId.canonical.z + 1,
-                                                            newId.canonical.x * 2 + x,
-                                                            newId.canonical.y * 2 + y);
-                            if (!id.equals(oldId)) {
-                                subdivIds.push(id);
-                            }
-                        }
-                    }
-                    return true;
-                }
-
-                // Remove identical IDs or children of existing IDs
-                return newId.equals(oldId) || newId.isChildOf(oldId);
-            }));
+        // Remove higher zoom new IDs that overlap with other new IDs
+        const nonOverlappingIds = [];
 
-            if (subdivIds.length > 0) {
-                out.push(...subdivIds);
-            } else {
-                finished = true;
+        for (const id of out) {
+            if (!out.some(ancestorCandidate => id.isChildOf(ancestorCandidate))) {
+                nonOverlappingIds.push(id);
             }
         }
 
+        // Remove new IDs that overlap with old IDs
+        out = nonOverlappingIds.filter(newId => !coveringTiles.some(oldId => {
+            if (newId.overscaledZ < maxZoom && oldId.isChildOf(newId)) {
+                return true;
+            }
+            // Remove identical IDs or children of existing IDs
+            return newId.equals(oldId) || newId.isChildOf(oldId);
+        }));
+
         return out;
     }
 

src/source/source_cache.js

@@ -517,6 +517,7 @@ class SourceCache extends Evented {
         this._coveredTiles = {};
 
         let idealTileIDs;
+
         if (!this.used && !this.usedForTerrain) {
             idealTileIDs = [];
         } else if (this._source.tileID) {
@@ -537,17 +538,18 @@ class SourceCache extends Evented {
             }
         }
 
-        if (idealTileIDs.length > 0 && this.castsShadows && directionalLight && this.transform.projection.name !== 'globe') {
-            // Extend the set of ideal tiles with potential shadow casters
+        if (idealTileIDs.length > 0 && this.castsShadows &&
+            directionalLight && this.transform.projection.name !== 'globe' &&
+            !this.usedForTerrain && !isRasterType(this._source.type)) {
+            // compute desired max zoom level
             const coveringZoom = transform.coveringZoomLevel({
                 tileSize: tileSize || this._source.tileSize,
                 roundZoom: this._source.roundZoom && !updateForTerrain
             });
+            const idealZoom = Math.min(coveringZoom, this._source.maxzoom);
 
-            const idealZoom = Math.min(coveringZoom, this._source.minzoom);
-
+            // find shadowCasterTiles
             const shadowCasterTileIDs = transform.extendTileCoverForShadows(idealTileIDs, directionalLight, idealZoom);
-
             for (const id of shadowCasterTileIDs) {
                 this._shadowCasterTiles[id.key] = true;
                 idealTileIDs.push(id);

src/source/tile_id.js

@@ -108,6 +108,7 @@ export class OverscaledTileID {
         // We're first testing for z == 0, to avoid a 32 bit shift, which is undefined.
         return parent.overscaledZ === 0 || (
             parent.overscaledZ < this.overscaledZ &&
+                parent.canonical.z < this.canonical.z &&
                 parent.canonical.x === (this.canonical.x >> zDifference) &&
                 parent.canonical.y === (this.canonical.y >> zDifference));
     }

src/source/worker_tile.js

@@ -122,12 +122,21 @@ class WorkerTile {
 
             let anySymbolLayers = false;
             let anyOtherLayers = false;
+            let any3DLayer = false;
+
             for (const family of layerFamilies[sourceLayerId]) {
                 if (family[0].type === 'symbol') {
                     anySymbolLayers = true;
                 } else {
                     anyOtherLayers = true;
                 }
+                if (family[0].is3D() && family[0].type !== 'model') {
+                    any3DLayer = true;
+                }
+            }
+
+            if (this.extraShadowCaster && !any3DLayer) {
+                continue;
             }
 
             if (this.isSymbolTile === true && !anySymbolLayers) {

test/unit/geo/transform.test.js

@@ -577,10 +577,6 @@ test('transform', (t) => {
             t.strictSame(shadowCasterTiles, [
                 new OverscaledTileID(3, -1, 3, 7, 0),
                 new OverscaledTileID(3, -1, 3, 7, 1),
-                new OverscaledTileID(4, 0, 4, 0, 3),
-                new OverscaledTileID(4, 0, 4, 2, 3),
-                new OverscaledTileID(4, 0, 4, 3, 2),
-                new OverscaledTileID(4, 0, 4, 3, 3),
                 new OverscaledTileID(5, 0, 5, 1, 8),
                 new OverscaledTileID(5, 0, 5, 2, 8),
                 new OverscaledTileID(5, 0, 5, 3, 8)