Add button capture to camera input system (#8918)

* draft

* Wasn't releasing properly fix

* pr comments(acuum and camelcase)

* Add better UX

* Update LinkComponent.ts

* Update LinkComponent.ts

* Fix input bouncing bug, Change to location navigation

* Create new engine hook for location moving in link component

---------

Co-authored-by: HexaField <joshfield999@gmail.com>

packages/client-core/i18n/en/common.json

@@ -10,6 +10,9 @@
   "tooltip": {
     "pressKey": "Press {{tip}} to {{message}}"
   },
+  "interactables": {
+    "link": "Click to follow"
+  },
   "toast": {
     "joined": "joined",
     "left": "left"

packages/client-core/i18n/en/editor.json

@@ -697,8 +697,7 @@
       "title": "link Component",
       "lbl-url": "Link URL",
       "lbl-navigateScene": "Scene navigation",
-      "lbl-projname": "Project Name",
-      "lbl-scenename": "Scene Name",
+      "lbl-locaiton": "Location Name",
       "description": "Adds an interactable that opens links or navigates scene"
     },
     "spotLight": {

packages/client-core/src/components/World/EngineHooks.tsx

@@ -61,6 +61,7 @@ import { TargetCameraRotationComponent } from '@etherealengine/engine/src/camera
 import { UndefinedEntity } from '@etherealengine/engine/src/ecs/classes/Entity'
 import { removeEntity } from '@etherealengine/engine/src/ecs/functions/EntityFunctions'
 import { WorldNetworkAction } from '@etherealengine/engine/src/networking/functions/WorldNetworkAction'
+import { LinkState } from '@etherealengine/engine/src/scene/components/LinkComponent'
 import { InstanceID } from '@etherealengine/engine/src/schemas/networking/instance.schema'
 import { projectsPath } from '@etherealengine/engine/src/schemas/projects/projects.schema'
 import { ComputedTransformComponent } from '@etherealengine/engine/src/transform/components/ComputedTransformComponent'
@@ -165,6 +166,30 @@ export const despawnSelfAvatar = () => {
   removeComponent(cameraEntity, TargetCameraRotationComponent)
 }
 
+export const useLinkTeleport = () => {
+  const linkState = useHookstate(getMutableState(LinkState))
+
+  useEffect(() => {
+    const location = linkState.location.value
+    if (!location) return
+
+    logger.info('Relocating to linked location.')
+
+    RouterState.navigate('/location/' + location)
+    LocationService.getLocationByName(location)
+
+    // shut down connection with existing world instance server
+    // leaving a world instance server will check if we are in a location media instance and shut that down too
+    leaveNetwork(NetworkState.worldNetwork as SocketWebRTCClientNetwork)
+
+    getMutableState(AppLoadingState).merge({
+      state: AppLoadingStates.START_STATE,
+      loaded: false
+    })
+    getMutableState(LinkState).location.set(undefined)
+  }, [linkState.location])
+}
+
 export const usePortalTeleport = () => {
   const engineState = useHookstate(getMutableState(EngineState))
   const locationState = useHookstate(getMutableState(LocationState))
@@ -225,6 +250,7 @@ export const useLoadEngineWithScene = ({ spectate }: Props = {}) => {
   useLoadEngine()
   useLocationSpawnAvatar(spectate)
   usePortalTeleport()
+  useLinkTeleport()
 
   useEffect(() => {
     if (engineState.sceneLoaded.value && appState.value !== AppLoadingStates.SUCCESS) {

packages/editor/src/components/properties/LinkNodeEditor.tsx

@@ -69,18 +69,11 @@ export const LinkNodeEditor: EditorComponentType = (props) => {
       </InputGroup>
       {linkComponent.sceneNav.value ? (
         <>
-          <InputGroup name="ProjectName" label={t('editor:properties.linkComp.lbl-projname')}>
+          <InputGroup name="Location" label={t('editor:properties.linkComp.lbl-locaiton')}>
             <ControlledStringInput
-              value={linkComponent.projectName.value}
-              onChange={updateProperty(LinkComponent, 'projectName')}
-              onRelease={commitProperty(LinkComponent, 'projectName')}
-            />
-          </InputGroup>
-          <InputGroup name="SceneName" label={t('editor:properties.linkComp.lbl-scenename')}>
-            <ControlledStringInput
-              value={linkComponent.sceneName.value}
-              onChange={updateProperty(LinkComponent, 'sceneName')}
-              onRelease={commitProperty(LinkComponent, 'sceneName')}
+              value={linkComponent.location.value}
+              onChange={updateProperty(LinkComponent, 'location')}
+              onRelease={commitProperty(LinkComponent, 'location')}
             />
           </InputGroup>
         </>

packages/engine/src/avatar/functions/avatarFunctions.ts

@@ -312,7 +312,7 @@ export function makeSkinnedMeshFromBoneData(bonesData) {
 export const getAvatarBoneWorldPosition = (entity: Entity, boneName: BoneNames, position: Vector3): boolean => {
   const avatarRigComponent = getOptionalComponent(entity, AvatarRigComponent)
   if (!avatarRigComponent) return false
-  const bone = avatarRigComponent.rig[boneName.toLowerCase()] as VRMHumanBone
+  const bone = avatarRigComponent.rig?.[boneName.toLowerCase()] as VRMHumanBone
   if (!bone) return false
   const el = bone.node.matrixWorld.elements
   position.set(el[12], el[13], el[14])

packages/engine/src/avatar/systems/AvatarInputSystem.ts

@@ -259,8 +259,14 @@ const execute = () => {
       onInteract()
     }
   }
-
-  for (const inputSourceEntity of nonCapturedInputSourceEntities) {
+  let inputEntities: Entity[] = nonCapturedInputSourceEntities
+  if (inputEntities.length === 0) {
+    inputEntities = inputSourceQuery().filter((entity) => {
+      const inputSource = getComponent(entity, InputSourceComponent)
+      if (controller.cameraEntity === inputSource.assignedButtonEntity) return true
+    })
+  }
+  for (const inputSourceEntity of inputEntities) {
     const inputSource = getComponent(inputSourceEntity, InputSourceComponent)
 
     const buttons = inputSource.buttons

packages/engine/src/camera/systems/CameraInputSystem.ts

@@ -180,13 +180,16 @@ const onKeyC = () => {
 
 const lastLookDelta = new Vector2()
 let lastMouseMoved = false
+const INPUT_CAPTURE_DELAY = 0.2
+let accumulator = 0
 
 const throttleHandleCameraZoom = throttle(handleCameraZoom, 30, { leading: true, trailing: false })
-
+let capturedInputSource: Entity | undefined = undefined
 const execute = () => {
   if (getState(XRState).xrFrame) return
 
   const deltaSeconds = getState(EngineState).deltaSeconds
+  accumulator += deltaSeconds
 
   const { localClientEntity } = Engine.instance
   if (!localClientEntity) return
@@ -195,13 +198,14 @@ const execute = () => {
 
   const avatarControllerEntities = avatarControllerQuery()
 
-  const nonCapturedInputSource = InputSourceComponent.nonCapturedInputSourceQuery()[0]
-  if (!nonCapturedInputSource) return
+  let inputSourceEntity = InputSourceComponent.nonCapturedInputSourceQuery()[0]
+  if (!inputSourceEntity && capturedInputSource) {
+    inputSourceEntity = capturedInputSource
+  }
 
   const avatarInputSettings = getState(AvatarInputSettingsState)
 
-  const inputSource = getComponent(nonCapturedInputSource, InputSourceComponent)
-
+  const inputSource = getComponent(inputSourceEntity, InputSourceComponent)
   const keys = inputSource.buttons
 
   if (keys.KeyV?.down) onKeyV()
@@ -242,7 +246,17 @@ const execute = () => {
         0.1
       )
     }
-
+    if (keys.PrimaryClick?.pressed) {
+      if (accumulator > INPUT_CAPTURE_DELAY) {
+        InputSourceComponent.captureButtons(cameraEntity)
+        capturedInputSource = inputSourceEntity
+        accumulator = 0
+      }
+    } else {
+      InputSourceComponent.releaseButtons()
+      capturedInputSource = undefined
+      accumulator = 0
+    }
     throttleHandleCameraZoom(cameraEntity, pointerState.scroll.y)
   }
 

packages/engine/src/interaction/functions/nonInteractUI.ts

@@ -0,0 +1,51 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import { WebLayer3D } from '@etherealengine/xrui'
+
+import { Entity } from '../../ecs/classes/Entity'
+import { addComponent, getComponent } from '../../ecs/functions/ComponentFunctions'
+import { NameComponent } from '../../scene/components/NameComponent'
+import { TransformComponent } from '../../transform/components/TransformComponent'
+import { XRUIComponent } from '../../xrui/components/XRUIComponent'
+import { createNonInteractiveModalView } from '../ui/nonInteractiveModalView'
+
+export function createNonInteractUI(entity: Entity, message: string) {
+  const ui = createNonInteractiveModalView(entity, message)
+  const nameComponent = getComponent(entity, NameComponent)
+  addComponent(ui.entity, NameComponent, 'ui-' + message + '-' + nameComponent)
+
+  const xrui = getComponent(ui.entity, XRUIComponent)
+  xrui.rootLayer.traverseLayersPreOrder((layer: WebLayer3D) => {
+    const mat = layer.contentMesh.material as THREE.MeshBasicMaterial
+    mat.transparent = true
+  })
+  const transform = getComponent(ui.entity, TransformComponent)
+  transform.scale.setScalar(1)
+
+  return ui
+}
+
+export const updateNonInteractUI = (entity: Entity, xrui: ReturnType<typeof createNonInteractUI>) => {}

packages/engine/src/interaction/systems/InteractiveSystem.ts

@@ -36,6 +36,7 @@ import { Engine } from '../../ecs/classes/Engine'
 import { EngineState } from '../../ecs/classes/EngineState'
 import { Entity } from '../../ecs/classes/Entity'
 import { defineQuery, getComponent, setComponent } from '../../ecs/functions/ComponentFunctions'
+import { removeEntity } from '../../ecs/functions/EntityFunctions'
 import { defineSystem } from '../../ecs/functions/SystemFunctions'
 import {
   DistanceFromCameraComponent,
@@ -100,7 +101,13 @@ export const onInteractableUpdate = (entity: Entity, xrui: ReturnType<typeof cre
 }
 
 export const getInteractiveUI = (entity: Entity) => InteractiveUI.get(entity)
-export const removeInteractiveUI = (entity: Entity) => InteractiveUI.delete(entity)
+export const removeInteractiveUI = (entity: Entity) => {
+  if (InteractiveUI.has(entity)) {
+    const { update, xrui } = getInteractiveUI(entity)!
+    removeEntity(xrui.entity)
+    InteractiveUI.delete(entity)
+  }
+}
 
 export const addInteractableUI = (
   entity: Entity,

packages/engine/src/interaction/ui/nonInteractiveModalView.tsx

@@ -0,0 +1,75 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import { createState } from '@hookstate/core'
+import React from 'react'
+
+import { Entity } from '../../ecs/classes/Entity'
+import { createXRUI } from '../../xrui/functions/createXRUI'
+import { useXRUIState } from '../../xrui/functions/useXRUIState'
+
+export interface ModalState {
+  message: string
+}
+
+export const createNonInteractiveModalView = (entity: Entity, message: string) => {
+  const ui = createXRUI(
+    nonInteractiveModalView,
+    createState({
+      message
+    } as ModalState),
+    { interactable: false }
+  )
+  return ui
+}
+
+export const nonInteractiveModalView = () => {
+  const modalState = useXRUIState<ModalState>()
+  return (
+    <div className={'modal'}>
+      <div className="hint" xr-layer="true" xr-pixel-ratio="1">
+        <div>{modalState.message.value}</div>
+      </div>
+      <link href="https://fonts.googleapis.com/css?family=Lato:400" rel="stylesheet" type="text/css" />
+      <style>
+        {`
+        .modal {
+          font-size: 60px;
+          background-color: #000d;
+          color: white;
+          font-family: 'Lato', sans-serif;
+          font-weight: 400;
+          border: 10px solid white;
+          border-radius: 50px;
+          padding: 20px;
+          margin: 60px;
+          width: 400px;
+          text-align: center;
+        }
+      `}
+      </style>
+    </div>
+  )
+}