Implement Lazy Picking for POINTERMOVE (#13044)

* Lazy Pick on Move

* Change to account for undefined

* cameras using DSM

* Check for reasons to pick before picking

* Addressed some feedback (missing meshUnderPointer)

* Modified get meshUnderPointer

* Add frame awareness picking for move

* Caching Test

* format

* import fix

* Removed test code

* PR Feedback

* fixed import

* removed gesture recognizer and reverted inputs

* Revert videoDome because it uses picking info

* formatting

* Changed to just MOVE Lazy Picking

* Format

* Removed comment

* PR Feedback

* Re-added meshunderpointer code back

* More feedback

* Comment

* PR Feedback part 1

* comment

packages/dev/core/src/Actions/actionManager.ts

@@ -274,6 +274,7 @@ export class ActionManager extends AbstractActionManager {
         }
 
         this.actions.push(action);
+        this.getScene()._registeredActions++;
 
         if (ActionManager.Triggers[action.trigger]) {
             ActionManager.Triggers[action.trigger]++;
@@ -301,6 +302,7 @@ export class ActionManager extends AbstractActionManager {
                 delete ActionManager.Triggers[action.trigger];
             }
             action._actionManager = null;
+            this.getScene()._registeredActions--;
             return true;
         }
         return false;

packages/dev/core/src/Collisions/pickingInfo.ts

@@ -12,9 +12,6 @@ declare type Ray = import("../Culling/ray").Ray;
  * @see https://doc.babylonjs.com/divingDeeper/mesh/interactions/picking_collisions
  */
 export class PickingInfo {
-    /** @internal */
-    public _pickingUnavailable = false;
-
     /**
      * If the pick collided with an object
      */

packages/dev/core/src/Culling/ray.ts

@@ -925,6 +925,12 @@ Scene.prototype.pickWithBoundingInfo = function (
     return result;
 };
 
+Object.defineProperty(Scene.prototype, "_pickingAvailable", {
+    get: () => true,
+    enumerable: false,
+    configurable: false,
+});
+
 Scene.prototype.pick = function (
     x: number,
     y: number,

packages/dev/core/src/Events/pointerEvents.ts

@@ -1,7 +1,8 @@
 import type { Nullable } from "../types";
 import { Vector2 } from "../Maths/math.vector";
 import type { PickingInfo } from "../Collisions/pickingInfo";
-import type { IMouseEvent } from "./deviceInputEvents";
+import type { IMouseEvent, IPointerEvent } from "./deviceInputEvents";
+import type { InputManager } from "../Inputs/scene.inputManager";
 
 declare type Ray = import("../Culling/ray").Ray;
 
@@ -109,21 +110,42 @@ export class PointerInfoPre extends PointerInfoBase {
  * The event member is an instance of PointerEvent for all types except PointerWheel and is of type MouseWheelEvent when type equals PointerWheel. The different event types can be found in the PointerEventTypes class.
  */
 export class PointerInfo extends PointerInfoBase {
+    private _pickInfo: Nullable<PickingInfo>;
+    private _inputManager: Nullable<InputManager>;
+
+    /**
+     * Defines the picking info associated with this PointerInfo object (if applicable)
+     */
+    public get pickInfo(): Nullable<PickingInfo> {
+        if (!this._pickInfo) {
+            this._generatePickInfo();
+        }
+
+        return this._pickInfo;
+    }
     /**
      * Instantiates a PointerInfo to store pointer related info to the onPointerObservable event.
      * @param type Defines the type of event (PointerEventTypes)
      * @param event Defines the related dom event
-     * @param pickInfo Defines the picking info associated to the info (if any)\
+     * @param pickInfo Defines the picking info associated to the info (if any)
+     * @param inputManager Defines the InputManager to use if there is no pickInfo
      */
-    constructor(
-        type: number,
-        event: IMouseEvent,
-        /**
-         * Defines the picking info associated to the info (if any)\
-         */
-        public pickInfo: Nullable<PickingInfo>
-    ) {
+    constructor(type: number, event: IMouseEvent, pickInfo: Nullable<PickingInfo>, inputManager: Nullable<InputManager> = null) {
         super(type, event);
+        this._pickInfo = pickInfo;
+        this._inputManager = inputManager;
+    }
+
+    /**
+     * Generates the picking info if needed
+     */
+    /** @internal */
+    public _generatePickInfo(): void {
+        if (this._inputManager) {
+            this._pickInfo = this._inputManager._pickMove((this.event as IPointerEvent).pointerId);
+            this._inputManager._setRayOnPointerInfo(this._pickInfo, this.event);
+            this._inputManager = null;
+        }
     }
 }
 

packages/dev/core/src/Inputs/scene.inputManager.ts

@@ -105,6 +105,7 @@ export class InputManager {
     private _previousStartingPointerTime = 0;
     private _pointerCaptures: { [pointerId: number]: boolean } = {};
     private _meshUnderPointerId: { [pointerId: number]: Nullable<AbstractMesh> } = {};
+    private _movePointerInfo: Nullable<PointerInfo> = null;
 
     // Keyboard
     private _onKeyDown: (evt: IKeyboardEvent) => void;
@@ -129,6 +130,13 @@ export class InputManager {
      * @returns Mesh that the pointer is pointer is hovering over
      */
     public get meshUnderPointer(): Nullable<AbstractMesh> {
+        if (this._movePointerInfo) {
+            // Because _pointerOverMesh is populated as part of _pickMove, we need to force a pick to update it.
+            // Calling _pickMove calls _setCursorAndPointerOverMesh which calls setPointerOverMesh
+            this._movePointerInfo._generatePickInfo();
+            // Once we have what we need, we can clear _movePointerInfo because we don't need it anymore
+            this._movePointerInfo = null;
+        }
         return this._pointerOverMesh;
     }
 
@@ -201,45 +209,40 @@ export class InputManager {
             }
         }
 
-        const isMeshPicked = pickResult && pickResult.hit && pickResult.pickedMesh ? true : false;
-        if (isMeshPicked) {
-            scene.setPointerOverMesh(pickResult!.pickedMesh, evt.pointerId, pickResult);
-
-            if (!scene.doNotHandleCursors && canvas && this._pointerOverMesh) {
-                const actionManager = this._pointerOverMesh._getActionManagerForTrigger();
-                if (actionManager && actionManager.hasPointerTriggers) {
-                    canvas.style.cursor = actionManager.hoverCursor || scene.hoverCursor;
-                }
-            }
-        } else {
-            scene.setPointerOverMesh(null, evt.pointerId, pickResult);
-        }
+        this._setCursorAndPointerOverMesh(pickResult, evt.pointerId, scene);
 
         for (const step of scene._pointerMoveStage) {
+            const isMeshPicked = pickResult?.pickedMesh ? true : false;
             pickResult = step.action(this._unTranslatedPointerX, this._unTranslatedPointerY, pickResult, isMeshPicked, canvas);
         }
 
-        if (pickResult) {
-            const type = evt.inputIndex >= PointerInput.MouseWheelX && evt.inputIndex <= PointerInput.MouseWheelZ ? PointerEventTypes.POINTERWHEEL : PointerEventTypes.POINTERMOVE;
+        const type = evt.inputIndex >= PointerInput.MouseWheelX && evt.inputIndex <= PointerInput.MouseWheelZ ? PointerEventTypes.POINTERWHEEL : PointerEventTypes.POINTERMOVE;
 
-            if (scene.onPointerMove) {
-                scene.onPointerMove(evt, pickResult, type);
-            }
+        if (scene.onPointerMove && pickResult) {
+            scene.onPointerMove(evt, pickResult, type);
+        }
 
-            if (scene.onPointerObservable.hasObservers()) {
-                const pi = new PointerInfo(type, evt, pickResult);
-                this._setRayOnPointerInfo(pi);
-                scene.onPointerObservable.notifyObservers(pi, type);
-            }
+        let pointerInfo: PointerInfo;
+        if (pickResult) {
+            pointerInfo = new PointerInfo(type, evt, pickResult);
+            this._setRayOnPointerInfo(pickResult, evt);
+        } else {
+            pointerInfo = new PointerInfo(type, evt, null, this);
+            this._movePointerInfo = pointerInfo;
+        }
+
+        if (scene.onPointerObservable.hasObservers()) {
+            scene.onPointerObservable.notifyObservers(pointerInfo, type);
         }
     }
 
     // Pointers handling
-    private _setRayOnPointerInfo(pointerInfo: PointerInfo) {
+    /** @internal */
+    public _setRayOnPointerInfo(pickInfo: Nullable<PickingInfo>, event: IMouseEvent) {
         const scene = this._scene;
-        if (pointerInfo.pickInfo && !pointerInfo.pickInfo._pickingUnavailable) {
-            if (!pointerInfo.pickInfo.ray) {
-                pointerInfo.pickInfo.ray = scene.createPickingRay(pointerInfo.event.offsetX, pointerInfo.event.offsetY, Matrix.Identity(), scene.activeCamera);
+        if (pickInfo && scene._pickingAvailable) {
+            if (!pickInfo.ray) {
+                pickInfo.ray = scene.createPickingRay(event.offsetX, event.offsetY, Matrix.Identity(), scene.activeCamera);
             }
         }
     }
@@ -263,6 +266,41 @@ export class InputManager {
         }
     }
 
+    /** @internal */
+    public _pickMove(pointerId: number): Nullable<PickingInfo> {
+        const scene = this._scene;
+        const pickResult = scene.pick(
+            this._unTranslatedPointerX,
+            this._unTranslatedPointerY,
+            scene.pointerMovePredicate,
+            false,
+            scene.cameraToUseForPointers,
+            scene.pointerMoveTrianglePredicate
+        );
+
+        this._setCursorAndPointerOverMesh(pickResult, pointerId, scene);
+
+        return pickResult;
+    }
+
+    private _setCursorAndPointerOverMesh(pickResult: Nullable<PickingInfo>, pointerId: number, scene: Scene) {
+        const engine = scene.getEngine();
+        const canvas = engine.getInputElement();
+
+        if (pickResult?.pickedMesh) {
+            this.setPointerOverMesh(pickResult.pickedMesh, pointerId, pickResult);
+
+            if (!scene.doNotHandleCursors && canvas && this._pointerOverMesh) {
+                const actionManager = this._pointerOverMesh._getActionManagerForTrigger();
+                if (actionManager && actionManager.hasPointerTriggers) {
+                    canvas.style.cursor = actionManager.hoverCursor || scene.hoverCursor;
+                }
+            }
+        } else {
+            this.setPointerOverMesh(null, pointerId, pickResult);
+        }
+    }
+
     /**
      * Use this method to simulate a pointer move on a mesh
      * The pickResult parameter can be obtained from a scene.pick or scene.pickWithRay
@@ -359,7 +397,7 @@ export class InputManager {
 
             if (scene.onPointerObservable.hasObservers()) {
                 const pi = new PointerInfo(type, evt, pickResult);
-                this._setRayOnPointerInfo(pi);
+                this._setRayOnPointerInfo(pickResult, evt);
                 scene.onPointerObservable.notifyObservers(pi, type);
             }
         }
@@ -412,7 +450,7 @@ export class InputManager {
                 if (clickInfo.singleClick && !clickInfo.ignore && scene.onPointerObservable.hasObservers()) {
                     const type = PointerEventTypes.POINTERPICK;
                     const pi = new PointerInfo(type, evt, pickResult);
-                    this._setRayOnPointerInfo(pi);
+                    this._setRayOnPointerInfo(pickResult, evt);
                     scene.onPointerObservable.notifyObservers(pi, type);
                 }
             }
@@ -454,15 +492,15 @@ export class InputManager {
                 }
                 if (type) {
                     const pi = new PointerInfo(type, evt, pickResult);
-                    this._setRayOnPointerInfo(pi);
+                    this._setRayOnPointerInfo(pickResult, evt);
                     scene.onPointerObservable.notifyObservers(pi, type);
                 }
             }
 
             if (!clickInfo.ignore) {
                 type = PointerEventTypes.POINTERUP;
                 const pi = new PointerInfo(type, evt, pickResult);
-                this._setRayOnPointerInfo(pi);
+                this._setRayOnPointerInfo(pickResult, evt);
                 scene.onPointerObservable.notifyObservers(pi, type);
             }
         }
@@ -508,9 +546,10 @@ export class InputManager {
         // Because this is only called from _initClickEvent, which is called in _onPointerUp, we'll use the pointerUpPredicate for the pick call
         this._initActionManager = (act: Nullable<AbstractActionManager>): Nullable<AbstractActionManager> => {
             if (!this._meshPickProceed) {
-                const pickResult = scene.skipPointerUpPicking
-                    ? null
-                    : scene.pick(this._unTranslatedPointerX, this._unTranslatedPointerY, scene.pointerUpPredicate, false, scene.cameraToUseForPointers);
+                const pickResult =
+                    scene.skipPointerUpPicking || (scene._registeredActions === 0 && !scene.onPointerObservable.hasObservers())
+                        ? null
+                        : scene.pick(this._unTranslatedPointerX, this._unTranslatedPointerY, scene.pointerUpPredicate, false, scene.cameraToUseForPointers);
                 this._currentPickResult = pickResult;
                 if (pickResult) {
                     act = pickResult.hit && pickResult.pickedMesh ? pickResult.pickedMesh._getActionManagerForTrigger() : null;
@@ -686,16 +725,7 @@ export class InputManager {
                     (!scene.cameraToUseForPointers || (scene.cameraToUseForPointers.layerMask & mesh.layerMask) !== 0);
             }
 
-            // Meshes
-            const pickResult = scene.pick(
-                this._unTranslatedPointerX,
-                this._unTranslatedPointerY,
-                scene.pointerMovePredicate,
-                false,
-                scene.cameraToUseForPointers,
-                scene.pointerMoveTrianglePredicate
-            );
-
+            const pickResult = scene._registeredActions > 0 ? this._pickMove((evt as IPointerEvent).pointerId) : null;
             this._processPointerMove(pickResult, evt as IPointerEvent);
         };
 
@@ -746,7 +776,7 @@ export class InputManager {
             // Meshes
             this._pickedDownMesh = null;
             let pickResult;
-            if (scene.skipPointerDownPicking) {
+            if (scene.skipPointerDownPicking || (scene._registeredActions === 0 && !scene.onPointerObservable.hasObservers())) {
                 pickResult = new PickingInfo();
             } else {
                 pickResult = scene.pick(this._unTranslatedPointerX, this._unTranslatedPointerY, scene.pointerDownPredicate, false, scene.cameraToUseForPointers);
@@ -984,7 +1014,7 @@ export class InputManager {
      * @returns a Mesh or null if no mesh is under the pointer
      */
     public getPointerOverMesh(): Nullable<AbstractMesh> {
-        return this._pointerOverMesh;
+        return this.meshUnderPointer;
     }
 
     /**

packages/dev/core/src/scene.ts

@@ -3787,6 +3787,7 @@ export class Scene extends AbstractScene implements IAnimatable, IClipPlanesHold
 
     /** @internal */
     public _activeMeshesFrozen = false;
+    /** @internal */
     public _activeMeshesFrozenButKeepClipping = false;
     private _skipEvaluateActiveMeshesCompletely = false;
 
@@ -4970,6 +4971,14 @@ export class Scene extends AbstractScene implements IAnimatable, IClipPlanesHold
         throw _WarnImport("Ray");
     }
 
+    /** @internal */
+    public get _pickingAvailable(): boolean {
+        return false;
+    }
+
+    /** @internal */
+    public _registeredActions: number = 0;
+
     /** Launch a ray to try to pick a mesh in the scene
      * @param x position on screen
      * @param y position on screen
@@ -4988,9 +4997,7 @@ export class Scene extends AbstractScene implements IAnimatable, IClipPlanesHold
         trianglePredicate?: TrianglePickingPredicate
     ): Nullable<PickingInfo> {
         // Dummy info if picking as not been imported
-        const pi = new PickingInfo();
-        pi._pickingUnavailable = true;
-        return pi;
+        return new PickingInfo();
     }
 
     /** Launch a ray to try to pick a mesh in the scene using only bounding information of the main mesh (not using submeshes)
@@ -5003,9 +5010,7 @@ export class Scene extends AbstractScene implements IAnimatable, IClipPlanesHold
      */
     public pickWithBoundingInfo(x: number, y: number, predicate?: (mesh: AbstractMesh) => boolean, fastCheck?: boolean, camera?: Nullable<Camera>): Nullable<PickingInfo> {
         // Dummy info if picking as not been imported
-        const pi = new PickingInfo();
-        pi._pickingUnavailable = true;
-        return pi;
+        return new PickingInfo();
     }
 
     /** Use the given ray to pick a mesh in the scene