From 706d7bcde5c42d7af063e383d85d6758fcc96e23 Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Mon, 4 Dec 2023 11:45:34 +0200 Subject: [PATCH] WebXR Mesh Detection (#5791) * WebXR Mesh Detection * ts issues * lint * minor changes based on PR's comments --- .../src/examples/xr/ar-mesh-detection.mjs | 277 ++++++++++++++++++ examples/src/examples/xr/index.mjs | 1 + src/framework/xr/xr-manager.js | 18 ++ src/framework/xr/xr-mesh-detection.js | 195 ++++++++++++ src/framework/xr/xr-mesh.js | 161 ++++++++++ 5 files changed, 652 insertions(+) create mode 100644 examples/src/examples/xr/ar-mesh-detection.mjs create mode 100644 src/framework/xr/xr-mesh-detection.js create mode 100644 src/framework/xr/xr-mesh.js diff --git a/examples/src/examples/xr/ar-mesh-detection.mjs b/examples/src/examples/xr/ar-mesh-detection.mjs new file mode 100644 index 00000000000..b15f24f3447 --- /dev/null +++ b/examples/src/examples/xr/ar-mesh-detection.mjs @@ -0,0 +1,277 @@ +import * as pc from 'playcanvas'; + +/** + * @typedef {import('../../options.mjs').ExampleOptions} ExampleOptions + * @param {import('../../options.mjs').ExampleOptions} options - The example options. + * @returns {Promise} The example application. + */ +async function example({ canvas }) { + /** + * @param {string} msg - The message. + */ + const message = function (msg) { + /** @type {HTMLDivElement} */ + let el = document.querySelector('.message'); + if (!el) { + el = document.createElement('div'); + el.classList.add('message'); + el.style.position = 'absolute'; + el.style.bottom = '96px'; + el.style.right = '0'; + el.style.padding = '8px 16px'; + el.style.fontFamily = 'Helvetica, Arial, sans-serif'; + el.style.color = '#fff'; + el.style.backgroundColor = 'rgba(0, 0, 0, 0.5)'; + document.body.append(el); + } + el.textContent = msg; + }; + + const assets = { + font: new pc.Asset('font', 'font', { url: assetPath + 'fonts/courier.json' }) + }; + + const app = new pc.Application(canvas, { + mouse: new pc.Mouse(canvas), + touch: new pc.TouchDevice(canvas), + keyboard: new pc.Keyboard(window), + graphicsDeviceOptions: { alpha: true } + }); + + app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW); + app.setCanvasResolution(pc.RESOLUTION_AUTO); + + // Ensure canvas is resized when window changes size + const resize = () => app.resizeCanvas(); + window.addEventListener('resize', resize); + app.on('destroy', () => { + window.removeEventListener('resize', resize); + }); + + // use device pixel ratio + app.graphicsDevice.maxPixelRatio = window.devicePixelRatio; + + const assetListLoader = new pc.AssetListLoader(Object.values(assets), app.assets); + assetListLoader.load(() => { + app.start(); + + // create camera + const camera = new pc.Entity(); + camera.addComponent('camera', { + clearColor: new pc.Color(0, 0, 0, 0), + farClip: 10000 + }); + app.root.addChild(camera); + + const l = new pc.Entity(); + l.addComponent("light", { + type: "omni", + range: 20 + }); + camera.addChild(l); + + if (app.xr.supported) { + const activate = function () { + if (app.xr.isAvailable(pc.XRTYPE_AR)) { + camera.camera.startXr(pc.XRTYPE_AR, pc.XRSPACE_LOCALFLOOR, { + meshDetection: true, + callback: function (err) { + if (err) message("WebXR Immersive AR failed to start: " + err.message); + } + }); + } else { + message("Immersive AR is not available"); + } + }; + + app.mouse.on("mousedown", function () { + if (!app.xr.active) + activate(); + }); + + if (app.touch) { + app.touch.on("touchend", function (evt) { + if (!app.xr.active) { + // if not in VR, activate + activate(); + } else { + // otherwise reset camera + camera.camera.endXr(); + } + + evt.event.preventDefault(); + evt.event.stopPropagation(); + }); + } + + // end session by keyboard ESC + app.keyboard.on('keydown', function (evt) { + if (evt.key === pc.KEY_ESCAPE && app.xr.active) { + app.xr.end(); + } + }); + + app.xr.on('start', function () { + message("Immersive AR session has started"); + + // Trigger manual room capture + // With a delay due to some issues on Quest 3 triggering immediately + setTimeout(() => { + app.xr.initiateRoomCapture((err) => { + if (err) console.log(err); + }); + }, 500); + }); + app.xr.on('end', function () { + message("Immersive AR session has ended"); + }); + app.xr.on('available:' + pc.XRTYPE_AR, function (available) { + if (available) { + if (app.xr.meshDetection.supported) { + message("Touch screen to start AR session and look at the floor or walls"); + } else { + message("AR Mesh Detection is not supported"); + } + } else { + message("Immersive AR is unavailable"); + } + }); + + const entities = new Map(); + + // materials + const materialDefault = new pc.StandardMaterial(); + + const materialGlobalMesh = new pc.StandardMaterial(); + materialGlobalMesh.blendType = pc.BLEND_ADDITIVEALPHA; + materialGlobalMesh.opacity = 0.2; + + const materialWireframe = new pc.StandardMaterial(); + materialWireframe.emissive = new pc.Color(1, 1, 1); + + // create entities for each XrMesh as they are added + app.xr.meshDetection.on('add', (xrMesh) => { + // solid mesh + const mesh = new pc.Mesh(app.graphicsDevice); + mesh.clear(true, false); + mesh.setPositions(xrMesh.vertices); + mesh.setNormals(pc.calculateNormals(xrMesh.vertices, xrMesh.indices)); + mesh.setIndices(xrMesh.indices); + mesh.update(pc.PRIMITIVE_TRIANGLES); + let material = xrMesh.label === 'global mesh' ? materialGlobalMesh : materialDefault; + const meshInstance = new pc.MeshInstance(mesh, material); + + // wireframe mesh + const meshWireframe = new pc.Mesh(app.graphicsDevice); + meshWireframe.clear(true, false); + meshWireframe.setPositions(xrMesh.vertices); + const indices = new Uint16Array(xrMesh.indices.length / 3 * 4); + for(let i = 0; i < xrMesh.indices.length; i += 3) { + const ind = i / 3 * 4; + indices[ind + 0] = xrMesh.indices[i + 0]; + indices[ind + 1] = xrMesh.indices[i + 1]; + indices[ind + 2] = xrMesh.indices[i + 1]; + indices[ind + 3] = xrMesh.indices[i + 2]; + } + meshWireframe.setIndices(indices); + meshWireframe.update(pc.PRIMITIVE_LINES); + const meshInstanceWireframe = new pc.MeshInstance(meshWireframe, materialWireframe); + meshInstanceWireframe.renderStyle = pc.RENDERSTYLE_WIREFRAME; + + // entity + const entity = new pc.Entity(); + entity.addComponent("render", { + meshInstances: [meshInstance, meshInstanceWireframe] + }); + app.root.addChild(entity); + entities.set(xrMesh, entity); + + // label + const label = new pc.Entity(); + label.setLocalPosition(0, 0, 0); + label.addComponent("element", { + pivot: new pc.Vec2(0.5, 0.5), + fontAsset: assets.font.id, + fontSize: 0.05, + text: xrMesh.label, + width: 1, + height: .1, + color: new pc.Color(1, 0, 0), + type: pc.ELEMENTTYPE_TEXT + }); + entity.addChild(label); + label.setLocalPosition(0, 0, .05); + entity.label = label; + + // transform + entity.setPosition(xrMesh.getPosition()); + entity.setRotation(xrMesh.getRotation()); + }); + + // when XrMesh is removed, destroy related entity + app.xr.meshDetection.on('remove', (xrMesh) => { + const entity = entities.get(xrMesh); + if (entity) { + entity.destroy(); + entities.delete(xrMesh); + } + }); + + const vec3A = new pc.Vec3(); + const vec3B = new pc.Vec3(); + const vec3C = new pc.Vec3(); + const transform = new pc.Mat4(); + + app.on('update', () => { + if (app.xr.active && app.xr.meshDetection.supported) { + // iterate through each XrMesh + for(let i = 0; i < app.xr.meshDetection.meshes.length; i++) { + const mesh = app.xr.meshDetection.meshes[i]; + + const entity = entities.get(mesh); + if (entity) { + // update entity transforms based on XrMesh + entity.setPosition(mesh.getPosition()); + entity.setRotation(mesh.getRotation()); + + // make sure label is looking at the camera + entity.label.lookAt(camera.getPosition()); + entity.label.rotateLocal(0, 180, 0); + } + + // render XrMesh gizmo axes + transform.setTRS(mesh.getPosition(), mesh.getRotation(), pc.Vec3.ONE); + vec3A.set(.2, 0, 0); + vec3B.set(0, .2, 0); + vec3C.set(0, 0, .2); + transform.transformPoint(vec3A, vec3A); + transform.transformPoint(vec3B, vec3B); + transform.transformPoint(vec3C, vec3C); + app.drawLine(mesh.getPosition(), vec3A, pc.Color.RED, false); + app.drawLine(mesh.getPosition(), vec3B, pc.Color.GREEN, false); + app.drawLine(mesh.getPosition(), vec3C, pc.Color.BLUE, false); + } + } + }); + + if (!app.xr.isAvailable(pc.XRTYPE_AR)) { + message("Immersive AR is not available"); + } else if (!app.xr.meshDetection.supported) { + message("AR Mesh Detection is not available"); + } else { + message("Touch screen to start AR session and look at the floor or walls"); + } + } else { + message("WebXR is not supported"); + } + }); + + return app; +} + +class ArMeshDetectionExample { + static CATEGORY = 'XR'; + static example = example; +} + +export { ArMeshDetectionExample }; diff --git a/examples/src/examples/xr/index.mjs b/examples/src/examples/xr/index.mjs index 27fd04e8eb5..95a9e9181a4 100644 --- a/examples/src/examples/xr/index.mjs +++ b/examples/src/examples/xr/index.mjs @@ -3,6 +3,7 @@ export * from "./ar-camera-color.mjs"; export * from "./ar-hit-test.mjs"; export * from "./ar-hit-test-anchors.mjs"; export * from "./ar-anchors-persistence.mjs"; +export * from "./ar-mesh-detection.mjs"; export * from "./ar-plane-detection.mjs"; export * from "./vr-basic.mjs"; export * from './vr-controllers.mjs'; diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index 707aa28a242..95ff60fdaa5 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -16,6 +16,7 @@ import { XrInput } from './xr-input.js'; import { XrLightEstimation } from './xr-light-estimation.js'; import { XrPlaneDetection } from './xr-plane-detection.js'; import { XrAnchors } from './xr-anchors.js'; +import { XrMeshDetection } from './xr-mesh-detection.js'; import { XrViews } from './xr-views.js'; /** @@ -133,6 +134,14 @@ class XrManager extends EventHandler { */ planeDetection; + /** + * Provides access to mesh detection capabilities. + * + * @type {XrMeshDetection} + * @ignore + */ + meshDetection; + /** * Provides access to Input Sources. * @@ -226,6 +235,7 @@ class XrManager extends EventHandler { this.hitTest = new XrHitTest(this); this.imageTracking = new XrImageTracking(this); this.planeDetection = new XrPlaneDetection(this); + this.meshDetection = new XrMeshDetection(this); this.input = new XrInput(this); this.lightEstimation = new XrLightEstimation(this); this.anchors = new XrAnchors(this); @@ -365,6 +375,8 @@ class XrManager extends EventHandler { * {@link XrImageTracking}. * @param {boolean} [options.planeDetection] - Set to true to attempt to enable * {@link XrPlaneDetection}. + * @param {boolean} [options.meshDetection] - Set to true to attempt to enable + * {@link XrMeshDetection}. * @param {XrErrorCallback} [options.callback] - Optional callback function called once session * is started. The callback has one argument Error - it is null if successfully started XR * session. @@ -439,6 +451,9 @@ class XrManager extends EventHandler { if (options.planeDetection) opts.optionalFeatures.push('plane-detection'); + + if (options.meshDetection) + opts.optionalFeatures.push('mesh-detection'); } if (this.domOverlay.supported && this.domOverlay.root) { @@ -872,6 +887,9 @@ class XrManager extends EventHandler { if (this.planeDetection.supported) this.planeDetection.update(frame); + + if (this.meshDetection.supported) + this.meshDetection.update(frame); } this.fire('update', frame); diff --git a/src/framework/xr/xr-mesh-detection.js b/src/framework/xr/xr-mesh-detection.js new file mode 100644 index 00000000000..e17d91f182e --- /dev/null +++ b/src/framework/xr/xr-mesh-detection.js @@ -0,0 +1,195 @@ +import { platform } from "../../core/platform.js"; +import { EventHandler } from "../../core/event-handler.js"; +import { XrMesh } from "./xr-mesh.js"; + +/** + * Mesh Detection provides the ability to detect real world meshes based on the + * scanning and reconstruction by the underlying AR system. + * + * ```javascript + * // start session with plane detection enabled + * app.xr.start(camera, pc.XRTYPE_AR, pc.XRSPACE_LOCALFLOOR, { + * meshDetection: true + * }); + * ``` + * + * ```javascript + * app.xr.meshDetection.on('add', function (mesh) { + * // new mesh been added + * }); + * ``` + * + * @category XR + */ +class XrMeshDetection extends EventHandler { + /** + * @type {import('./xr-manager.js').XrManager} + * @private + */ + _manager; + + /** + * @type {boolean} + * @private + */ + _supported = platform.browser && !!window.XRMesh; + + /** + * @type {boolean} + * @private + */ + _available = false; + + /** + * @type {Map} + * @private + */ + _index = new Map(); + + /** + * @type {XrMesh[]} + * @private + */ + _list = []; + + /** + * Create a new XrMeshDetection instance. + * + * @param {import('./xr-manager.js').XrManager} manager - WebXR Manager. + * @hideconstructor + */ + constructor(manager) { + super(); + + this._manager = manager; + + if (this._supported) { + this._manager.on('start', this._onSessionStart, this); + this._manager.on('end', this._onSessionEnd, this); + } + } + + /** + * Fired when mesh detection becomes available. + * + * @event XrMeshDetection#available + */ + + /** + * Fired when mesh detection becomes unavailable. + * + * @event XrMeshDetection#unavailable + */ + + /** + * Fired when new {@link XrMesh} is added to the list. + * + * @event XrMeshDetection#add + * @param {XrMesh} mesh - Mesh that has been added. + * @example + * app.xr.meshDetection.on('add', (mesh) => { + * // a new XrMesh has been added + * }); + */ + + /** + * Fired when a {@link XrMesh} is removed from the list. + * + * @event XrMeshDetection#remove + * @param {XrMesh} mesh - Mesh that has been removed. + * @example + * app.xr.meshDetection.on('remove', (mesh) => { + * // XrMesh has been removed + * }); + */ + + /** + * @param {XRFrame} frame - XRFrame from requestAnimationFrame callback. + * @ignore + */ + update(frame) { + if (!this._supported || !this._available) + return; + + // add meshes + for (const xrMesh of frame.detectedMeshes) { + let mesh = this._index.get(xrMesh); + if (!mesh) { + mesh = new XrMesh(this, xrMesh); + this._index.set(xrMesh, mesh); + this._list.push(mesh); + mesh.update(frame); + this.fire('add', mesh); + } else { + mesh.update(frame); + } + } + + // remove meshes + for (const mesh of this._index.values()) { + if (frame.detectedMeshes.has(mesh.xrMesh)) + continue; + + this._removeMesh(mesh); + } + } + + /** + * @param {XrMesh} mesh - XrMesh to remove. + * @private + */ + _removeMesh(mesh) { + this._index.delete(mesh.xrMesh); + this._list.splice(this._list.indexOf(mesh), 1); + mesh.destroy(); + this.fire('remove', mesh); + } + + /** @private */ + _onSessionStart() { + const available = this._manager.session.enabledFeatures.indexOf('mesh-detection') !== -1; + if (!available) return; + this._available = available; + this.fire('available'); + } + + /** @private */ + _onSessionEnd() { + if (!this._available) return; + this._available = false; + + for (const mesh of this._index.values()) + this._removeMesh(mesh); + + this.fire('unavailable'); + } + + /** + * True if Mesh Detection is supported. + * + * @type {boolean} + */ + get supported() { + return this._supported; + } + + /** + * True if Mesh Detection is available. This information is available only when session has started. + * + * @type {boolean} + */ + get available() { + return this._available; + } + + /** + * Array of {@link XrMesh} instances that contain transform, vertices and label information. + * + * @type {XrMesh[]|null} + */ + get meshes() { + return this._list; + } +} + +export { XrMeshDetection }; diff --git a/src/framework/xr/xr-mesh.js b/src/framework/xr/xr-mesh.js new file mode 100644 index 00000000000..35728114b71 --- /dev/null +++ b/src/framework/xr/xr-mesh.js @@ -0,0 +1,161 @@ +import { EventHandler } from "../../core/event-handler.js"; +import { Vec3 } from "../../core/math/vec3.js"; +import { Quat } from "../../core/math/quat.js"; + +/** + * Detected Mesh instance that provides its transform (position, rotation), + * triangles (vertices, indices) and its semantic label. Any of its properties can + * change during its lifetime. + * + * @category XR + */ +class XrMesh extends EventHandler { + /** + * @type {import('./xr-mesh-detection.js').XrMeshDetection} + * @private + */ + _meshDetection; + + /** + * @type {XRMesh} + * @private + */ + _xrMesh; + + /** + * @type {number} + * @private + */ + _lastChanged = 0; + + /** + * @type {Vec3} + * @private + */ + _position = new Vec3(); + + /** + * @type {Quat} + * @private + */ + _rotation = new Quat(); + + /** + * Create a new XrMesh instance. + * + * @param {import('./xr-mesh-detection.js').XrMeshDetection} meshDetection - Mesh Detection + * interface. + * @param {XRMesh} xrMesh - XRMesh that is instantiated by WebXR system. + * @hideconstructor + */ + constructor(meshDetection, xrMesh) { + super(); + + this._meshDetection = meshDetection; + this._xrMesh = xrMesh; + this._lastChanged = this._xrMesh.lastChangedTime; + } + + /** + * Fired when {@link XrMesh} is removed. + * + * @event XrMesh#remove + * @example + * mesh.once('remove', function () { + * // mesh is no longer available + * }); + */ + + /** + * Fired when {@link XrMesh} attributes such as vertices, indices and/or label have been changed. + * Position and rotation can change at any time without triggering a `change` event. + * + * @event XrMesh#change + * @example + * mesh.on('change', function () { + * // mesh attributes have been changed + * }); + */ + + /** + * @type {XRMesh} + * @ignore + */ + get xrMesh() { + return this._xrMesh; + } + + /** + * Semantic Label of a mesh that is provided by underlying system. + * Current list includes (but not limited to): https://github.com/immersive-web/semantic-labels/blob/master/labels.json + * + * @type {string} + */ + get label() { + return this._xrMesh.semanticLabel || ''; + } + + /** + * Float 32 array of mesh vertices. + * + * @type {Float32Array} + */ + get vertices() { + return this._xrMesh.vertices; + } + + /** + * Uint 32 array of mesh indices. + * + * @type {Uint32Array} + */ + get indices() { + return this._xrMesh.indices; + } + + /** @ignore */ + destroy() { + if (!this._xrMesh) return; + this._xrMesh = null; + this.fire('remove'); + } + + /** + * @param {XRFrame} frame - XRFrame from requestAnimationFrame callback. + * @ignore + */ + update(frame) { + const manager = this._meshDetection._manager; + const pose = frame.getPose(this._xrMesh.meshSpace, manager._referenceSpace); + if (pose) { + this._position.copy(pose.transform.position); + this._rotation.copy(pose.transform.orientation); + } + + // attributes have been changed + if (this._lastChanged !== this._xrMesh.lastChangedTime) { + this._lastChanged = this._xrMesh.lastChangedTime; + this.fire('change'); + } + } + + /** + * Get the world space position of a mesh. + * + * @returns {Vec3} The world space position of a mesh. + */ + getPosition() { + return this._position; + } + + /** + * Get the world space rotation of a mesh. + * + * @returns {Quat} The world space rotation of a mesh. + */ + getRotation() { + return this._rotation; + } +} + +export { XrMesh };