Support for Picker on WebGPU

examples/src/examples/graphics/area-picker.example.mjs

@@ -1,4 +1,3 @@
-// @config WEBGPU_DISABLED
 import * as pc from 'playcanvas';
 import { deviceType, rootPath } from '@examples/utils';
 
@@ -193,12 +192,6 @@ assetListLoader.load(() => {
         camera.setLocalPosition(40 * Math.sin(time), 0, 40 * Math.cos(time));
         camera.lookAt(pc.Vec3.ZERO);
 
-        // turn all previously highlighted meshes to black at the start of the frame
-        for (let h = 0; h < highlights.length; h++) {
-            highlightMaterial(highlights[h], pc.Color.BLACK);
-        }
-        highlights.length = 0;
-
         // Make sure the picker is the right size, and prepare it, which renders meshes into its render target
         if (picker) {
             picker.resize(canvas.clientWidth * pickerScale, canvas.clientHeight * pickerScale);
@@ -231,34 +224,51 @@ assetListLoader.load(() => {
             }
         ];
 
-        // process all areas
+        // process all areas every frame
+        const promises = [];
         for (let a = 0; a < areas.length; a++) {
             const areaPos = areas[a].pos;
             const areaSize = areas[a].size;
-            const color = areas[a].color;
 
             // display 2D rectangle around it
             drawRectangle(areaPos.x, areaPos.y, areaSize.x, areaSize.y);
 
             // get list of meshInstances inside the area from the picker
             // this scans the pixels inside the render target and maps the id value stored there into meshInstances
-            const selection = picker.getSelection(
+            // Note that this is an async function returning a promise. Store it in the promises array.
+            const promise = picker.getSelectionAsync(
                 areaPos.x * pickerScale,
                 areaPos.y * pickerScale,
                 areaSize.x * pickerScale,
                 areaSize.y * pickerScale
             );
 
-            // process all meshInstances it found - highlight them to appropriate color for the area
-            for (let s = 0; s < selection.length; s++) {
-                if (selection[s]) {
-                    /** @type {pc.StandardMaterial} */
-                    const material = selection[s].material;
-                    highlightMaterial(material, color);
-                    highlights.push(material);
+            promises.push(promise);
+        }
+
+        // when all promises are resolved, we can highlight the meshes
+        Promise.all(promises).then((results) => {
+
+            // turn off previously highlighted meshes
+            for (let h = 0; h < highlights.length; h++) {
+                highlightMaterial(highlights[h], pc.Color.BLACK);
+            }
+            highlights.length = 0;
+
+            // process the results
+            for (let i = 0; i < results.length; i++) {
+                const meshInstances = results[i];
+
+                for (let s = 0; s < meshInstances.length; s++) {
+                    if (meshInstances[s]) {
+                        /** @type {pc.StandardMaterial} */
+                        const material = meshInstances[s].material;
+                        highlightMaterial(material, areas[i].color);
+                        highlights.push(material);
+                    }
                 }
             }
-        }
+        });
     });
 });
 

examples/src/examples/misc/gizmos.example.mjs

@@ -1,5 +1,4 @@
 // @config DESCRIPTION <div style='text-align:center'><div>Translate (1), Rotate (2), Scale (3)</div><div>World/Local (X)</div><div>Perspective (P), Orthographic (O)</div></div>
-// @config WEBGPU_DISABLED
 import * as pc from 'playcanvas';
 import { data } from '@examples/observer';
 import { deviceType, rootPath, localImport } from '@examples/utils';
@@ -262,13 +261,14 @@ const onPointerDown = (/** @type {PointerEvent} */ e) => {
         picker.prepare(camera.camera, app.scene, pickerLayers);
     }
 
-    const selection = picker.getSelection(e.clientX - 1, e.clientY - 1, 2, 2);
-    if (!selection[0]) {
-        gizmoHandler.clear();
-        return;
-    }
+    picker.getSelectionAsync(e.clientX - 1, e.clientY - 1, 2, 2).then((selection) => {
+        if (!selection[0]) {
+            gizmoHandler.clear();
+            return;
+        }
 
-    gizmoHandler.add(selection[0].node, !e.ctrlKey && !e.metaKey);
+        gizmoHandler.add(selection[0].node, !e.ctrlKey && !e.metaKey);
+    });
 };
 window.addEventListener('pointerdown', onPointerDown);
 

src/framework/graphics/picker.js

@@ -11,8 +11,11 @@ import { Layer } from '../../scene/layer.js';
 import { getApplication } from '../globals.js';
 import { Debug } from '../../core/debug.js';
 import { RenderPassPicker } from './render-pass-picker.js';
+import { math } from '../../core/math/math.js';
+import { Vec4 } from '../../core/math/vec4.js';
 
 const tempSet = new Set();
+const _rect = new Vec4();
 
 /**
  * Picker object used to select mesh instances from screen coordinates.
@@ -31,6 +34,9 @@ class Picker {
     // mapping table from ids to meshInstances
     mapping = new Map();
 
+    // when the device is destroyed, this allows us to ignore async results
+    deviceValid = true;
+
     /**
      * Create a new Picker instance.
      *
@@ -55,11 +61,20 @@ class Picker {
         this.width = 0;
         this.height = 0;
         this.resize(width, height);
+
+        // handle the device getting destroyed
+        this.device.on('destroy', () => {
+            this.deviceValid = false;
+        });
     }
 
     /**
      * Return the list of mesh instances selected by the specified rectangle in the previously
-     * prepared pick buffer.The rectangle using top-left coordinate system.
+     * prepared pick buffer. The rectangle using top-left coordinate system.
+     *
+     * Note: This function is not supported on WebGPU. Use {@link Picker#getSelectionAsync} instead.
+     * Note: This function is blocks the main thread while reading pixels from GPU memory. It's
+     * recommended to use {@link Picker#getSelectionAsync} instead.
      *
      * @param {number} x - The left edge of the rectangle.
      * @param {number} y - The top edge of the rectangle.
@@ -76,50 +91,110 @@ class Picker {
      */
     getSelection(x, y, width = 1, height = 1) {
         const device = this.device;
+        if (device.isWebGPU) {
+            Debug.errorOnce('pc.Picker#getSelection is not supported on WebGPU, use pc.Picker#getSelectionAsync instead.');
+            return [];
+        }
 
         Debug.assert(typeof x !== 'object', `Picker.getSelection:param 'rect' is deprecated, use 'x, y, width, height' instead.`);
 
         y = this.renderTarget.height - (y + height);
-
-        // make sure we have nice numbers to work with
-        x = Math.floor(x);
-        y = Math.floor(y);
-        width = Math.floor(Math.max(width, 1));
-        height = Math.floor(Math.max(height, 1));
+        const rect = this.sanitizeRect(x, y, width, height);
 
         // read pixels from the render target
         device.setRenderTarget(this.renderTarget);
         device.updateBegin();
 
-        const pixels = new Uint8Array(4 * width * height);
-        device.readPixels(x, y, width, height, pixels);
+        const pixels = new Uint8Array(4 * rect.z * rect.w);
+        device.readPixels(rect.x, rect.y, rect.z, rect.w, pixels);
 
         device.updateEnd();
 
-        const mapping = this.mapping;
-        for (let i = 0; i < width * height; i++) {
-            const r = pixels[4 * i + 0];
-            const g = pixels[4 * i + 1];
-            const b = pixels[4 * i + 2];
-            const a = pixels[4 * i + 3];
-            const index = a << 24 | r << 16 | g << 8 | b;
-
-            // White is 'no selection'
-            if (index !== -1) {
-                tempSet.add(mapping.get(index));
-            }
+        return this.decodePixels(pixels, this.mapping);
+    }
+
+    /**
+     * Return the list of mesh instances selected by the specified rectangle in the previously
+     * prepared pick buffer. The rectangle uses top-left coordinate system.
+     *
+     * This method is asynchronous and does not block the execution.
+     *
+     * @param {number} x - The left edge of the rectangle.
+     * @param {number} y - The top edge of the rectangle.
+     * @param {number} [width] - The width of the rectangle. Defaults to 1.
+     * @param {number} [height] - The height of the rectangle. Defaults to 1.
+     * @returns {Promise<import('../../scene/mesh-instance.js').MeshInstance[]>} - Promise that
+     * resolves with an array of mesh instances that are in the selection.
+     * @example
+     * // Get the mesh instances at the rectangle with start at (10,20) and size of (5,5)
+     * picker.getSelectionAsync(10, 20, 5, 5).then((meshInstances) => {
+     *    console.log(meshInstances);
+     * });
+     */
+    getSelectionAsync(x, y, width = 1, height = 1) {
+
+        if (this.device?.isWebGL2) {
+            y = this.renderTarget.height - (y + height);
         }
+        const rect = this.sanitizeRect(x, y, width, height);
+
+        return this.renderTarget.colorBuffer.read(rect.x, rect.y, rect.z, rect.w, {
+            renderTarget: this.renderTarget,
+            immediate: true
+        }).then((pixels) => {
+            return this.decodePixels(pixels, this.mapping);
+        });
+    }
+
+    // sanitize the rectangle to make sure it;s inside the texture and does not use fractions
+    sanitizeRect(x, y, width, height) {
+        const maxWidth = this.renderTarget.width;
+        const maxHeight = this.renderTarget.height;
+        x = math.clamp(Math.floor(x), 0, maxWidth - 1);
+        y = math.clamp(Math.floor(y), 0, maxHeight - 1);
+        width = Math.floor(Math.max(width, 1));
+        width = Math.min(width, maxWidth - x);
+        height = Math.floor(Math.max(height, 1));
+        height = Math.min(height, maxHeight - y);
+        return _rect.set(x, y, width, height);
+    }
+
+    decodePixels(pixels, mapping) {
 
-        // return the content of the set as an array
         const selection = [];
-        tempSet.forEach(meshInstance => selection.push(meshInstance));
-        tempSet.clear();
+
+        // when we decode results from async calls, ignore them if the device is no longer valid
+        if (this.deviceValid) {
+
+            const count = pixels.length;
+            for (let i = 0; i < count; i += 4) {
+                const r = pixels[i + 0];
+                const g = pixels[i + 1];
+                const b = pixels[i + 2];
+                const a = pixels[i + 3];
+                const index = a << 24 | r << 16 | g << 8 | b;
+
+                // White is 'no selection'
+                if (index !== -1) {
+                    tempSet.add(mapping.get(index));
+                }
+            }
+
+            // return the content of the set as an array
+            tempSet.forEach((meshInstance) => {
+                if (meshInstance)
+                    selection.push(meshInstance);
+            });
+            tempSet.clear();
+        }
 
         return selection;
     }
 
     allocateRenderTarget() {
 
+        // TODO: Ideally we'd use a UINT32 texture format and avoid RGBA8 conversion, but WebGL2 does not
+        // support clearing render targets of this format, so we'd need a quad based clear solution.
         const colorBuffer = new Texture(this.device, {
             format: PIXELFORMAT_RGBA8,
             width: this.width,

src/framework/graphics/render-pass-picker.js

@@ -13,14 +13,22 @@ const lights = [[], [], []];
  * @ignore
  */
 class RenderPassPicker extends RenderPass {
-    // uniform for the mesh index encoded into rgba
-    pickColor = new Float32Array(4);
+    /** @type {import('../../platform/graphics/bind-group.js').BindGroup[]} */
+    viewBindGroups = [];
 
     constructor(device, renderer) {
         super(device);
         this.renderer = renderer;
     }
 
+    destroy() {
+        this.viewBindGroups.forEach((bg) => {
+            bg.defaultUniformBuffer.destroy();
+            bg.destroy();
+        });
+        this.viewBindGroups.length = 0;
+    }
+
     update(camera, scene, layers, mapping) {
         this.camera = camera;
         this.scene = scene;
@@ -30,16 +38,12 @@ class RenderPassPicker extends RenderPass {
 
     execute() {
         const device = this.device;
-        DebugGraphics.pushGpuMarker(device, 'PICKER');
 
         const { renderer, camera, scene, layers, mapping, renderTarget } = this;
         const srcLayers = scene.layers.layerList;
         const subLayerEnabled = scene.layers.subLayerEnabled;
         const isTransparent = scene.layers.subLayerList;
 
-        const pickColorId = device.scope.resolve('uColor');
-        const pickColor = this.pickColor;
-
         for (let i = 0; i < srcLayers.length; i++) {
             const srcLayer = srcLayers[i];
 
@@ -79,14 +83,19 @@ class RenderPassPicker extends RenderPass {
 
                     if (tempMeshInstances.length > 0) {
 
+                        // upload clustered lights uniforms
+                        const clusteredLightingEnabled = scene.clusteredLightingEnabled;
+                        if (clusteredLightingEnabled) {
+                            const lightClusters = renderer.worldClustersAllocator.empty;
+                            lightClusters.activate();
+                        }
+
                         renderer.setCameraUniforms(camera.camera, renderTarget);
+                        if (device.supportsUniformBuffers) {
+                            renderer.setupViewUniformBuffers(this.viewBindGroups, renderer.viewUniformFormat, renderer.viewBindGroupFormat, 1);
+                        }
+
                         renderer.renderForward(camera.camera, tempMeshInstances, lights, SHADER_PICK, (meshInstance) => {
-                            const miId = meshInstance.id;
-                            pickColor[0] = ((miId >> 16) & 0xff) / 255;
-                            pickColor[1] = ((miId >> 8) & 0xff) / 255;
-                            pickColor[2] = (miId & 0xff) / 255;
-                            pickColor[3] = ((miId >> 24) & 0xff) / 255;
-                            pickColorId.setValue(pickColor);
                             device.setBlendState(BlendState.NOBLEND);
                         });
 
@@ -97,8 +106,6 @@ class RenderPassPicker extends RenderPass {
                 }
             }
         }
-
-        DebugGraphics.popGpuMarker(device);
     }
 }