diff --git a/habitat-hitl/habitat_hitl/_internal/gui_application.py b/habitat-hitl/habitat_hitl/_internal/gui_application.py index 8b85744251..98afc42fab 100644 --- a/habitat-hitl/habitat_hitl/_internal/gui_application.py +++ b/habitat-hitl/habitat_hitl/_internal/gui_application.py @@ -7,6 +7,7 @@ import abc import math import time +from typing import List import magnum as mn from magnum.platform.glfw import Application @@ -32,13 +33,13 @@ def unproject(self, viewport_pos): class InputHandlerApplication(Application): def __init__(self, config): super().__init__(config) - self._gui_inputs = [] + self._gui_inputs: List[GuiInput] = [] - def add_gui_input(self, gui_input): + def add_gui_input(self, gui_input: GuiInput) -> None: self._gui_inputs.append(gui_input) def key_press_event(self, event: Application.KeyEvent) -> None: - key = MagnumKeyConverter.convert(event.key) + key = MagnumKeyConverter.convert_key(event.key) if key: for wrapper in self._gui_inputs: # If the key is already held, this is a repeat press event and we should @@ -48,7 +49,7 @@ def key_press_event(self, event: Application.KeyEvent) -> None: wrapper._key_down.add(key) def key_release_event(self, event: Application.KeyEvent) -> None: - key = MagnumKeyConverter.convert(event.key) + key = MagnumKeyConverter.convert_key(event.key) if key: for wrapper in self._gui_inputs: if key in wrapper._key_held: @@ -56,22 +57,22 @@ def key_release_event(self, event: Application.KeyEvent) -> None: wrapper._key_up.add(key) def mouse_press_event(self, event: Application.MouseEvent) -> None: - mouse_button = event.button - GuiInput.validate_mouse_button(mouse_button) - for wrapper in self._gui_inputs: - wrapper._mouse_button_held.add(mouse_button) - wrapper._mouse_button_down.add(mouse_button) + key = MagnumKeyConverter.convert_mouse_button(event.button) + if key: + for wrapper in self._gui_inputs: + # If the key is already held, this is a repeat press event and we should + # ignore it. + if key not in wrapper._mouse_button_held: + wrapper._mouse_button_held.add(key) + wrapper._mouse_button_down.add(key) def mouse_release_event(self, event: Application.MouseEvent) -> None: - mouse_button = event.button - GuiInput.validate_mouse_button(mouse_button) - for wrapper in self._gui_inputs: - # In theory, mouse_button should always be present in _mouse_button_held. - # In practice, we seem to get spurious release events due to the app - # losing focus (e.g. switching to VS code debugger while mouse-clicking) - if mouse_button in wrapper._mouse_button_held: - wrapper._mouse_button_held.remove(mouse_button) - wrapper._mouse_button_up.add(mouse_button) + key = MagnumKeyConverter.convert_mouse_button(event.button) + if key: + for wrapper in self._gui_inputs: + if key in wrapper._mouse_button_held: + wrapper._mouse_button_held.remove(key) + wrapper._mouse_button_up.add(key) def mouse_scroll_event(self, event: Application.MouseEvent) -> None: # shift+scroll is forced into x direction on mac, seemingly at OS level, diff --git a/habitat-hitl/habitat_hitl/core/gui_input.py b/habitat-hitl/habitat_hitl/core/gui_input.py index 184525759f..caf3eb660c 100644 --- a/habitat-hitl/habitat_hitl/core/gui_input.py +++ b/habitat-hitl/habitat_hitl/core/gui_input.py @@ -4,17 +4,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from habitat_hitl.core.key_mapping import KeyCode - - -class StubNSMeta(type): - def __getattr__(cls, name): - return None - - -# Stub version of Application.MouseEvent.Button -class StubMouseNS(metaclass=StubNSMeta): - pass +from habitat_hitl.core.key_mapping import KeyCode, MouseButton class GuiInput: @@ -25,7 +15,7 @@ class GuiInput: """ KeyNS = KeyCode - MouseNS = StubMouseNS + MouseNS = MouseButton def __init__(self): self._key_held = set() @@ -37,7 +27,7 @@ def __init__(self): self._mouse_button_down = set() self._mouse_button_up = set() self._relative_mouse_position = [0, 0] - self._mouse_scroll_offset = 0 + self._mouse_scroll_offset = 0.0 self._mouse_ray = None def validate_key(key): @@ -59,9 +49,7 @@ def get_key_up(self, key): return key in self._key_up def validate_mouse_button(mouse_button): - # if not do_agnostic_gui_input: - # assert isinstance(mouse_button, Application.MouseEvent.Button) - pass + assert isinstance(mouse_button, MouseButton) def get_mouse_button(self, mouse_button): GuiInput.validate_mouse_button(mouse_button) @@ -99,4 +87,4 @@ def on_frame_end(self): self._mouse_button_down.clear() self._mouse_button_up.clear() self._relative_mouse_position = [0, 0] - self._mouse_scroll_offset = 0 + self._mouse_scroll_offset = 0.0 diff --git a/habitat-hitl/habitat_hitl/core/key_mapping.py b/habitat-hitl/habitat_hitl/core/key_mapping.py index bf2e641b60..e1f8fb3204 100644 --- a/habitat-hitl/habitat_hitl/core/key_mapping.py +++ b/habitat-hitl/habitat_hitl/core/key_mapping.py @@ -69,6 +69,18 @@ class KeyCode(IntEnum, metaclass=KeyCodeMetaEnum): # fmt: on +class MouseButton(IntEnum, metaclass=KeyCodeMetaEnum): + """ + Mouse buttons available to control habitat-hitl. + """ + + # fmt: off + LEFT = 0 + RIGHT = 1 + MIDDLE = 2 + # fmt: on + + # On headless systems, we may be unable to import magnum.platform.glfw.Application. try: from magnum.platform.glfw import Application @@ -124,9 +136,22 @@ class KeyCode(IntEnum, metaclass=KeyCodeMetaEnum): # fmt: on } + magnum_mouse_keymap: Dict[Application.KeyEvent.Key, MouseButton] = { + # fmt: off + Application.MouseEvent.Button.LEFT : MouseButton.LEFT , + Application.MouseEvent.Button.RIGHT : MouseButton.RIGHT , + Application.MouseEvent.Button.MIDDLE : MouseButton.MIDDLE, + # fmt: on + } + class MagnumKeyConverter: - def convert(key: Any) -> Optional[KeyCode]: + def convert_key(key: Any) -> Optional[KeyCode]: if magnum_enabled and key in magnum_keymap: return magnum_keymap[key] return None + + def convert_mouse_button(button: Any) -> Optional[MouseButton]: + if magnum_enabled and button in magnum_mouse_keymap: + return magnum_mouse_keymap[button] + return None diff --git a/habitat-hitl/habitat_hitl/core/remote_client_state.py b/habitat-hitl/habitat_hitl/core/remote_client_state.py index 690cf1ac8c..a84227d0bb 100644 --- a/habitat-hitl/habitat_hitl/core/remote_client_state.py +++ b/habitat-hitl/habitat_hitl/core/remote_client_state.py @@ -156,10 +156,9 @@ def _update_input_state(self, client_states): input_json = ( client_state["input"] if "input" in client_state else None ) - # TODO: Add mouse support - # mouse_json = ( - # client_state["mouse"] if "mouse" in client_state else None - # ) + mouse_json = ( + client_state["mouse"] if "mouse" in client_state else None + ) if input_json is not None: for button in input_json["buttonDown"]: @@ -171,6 +170,23 @@ def _update_input_state(self, client_states): continue self._gui_input._key_up.add(KeyCode(button)) + if mouse_json is not None: + mouse_buttons = mouse_json["buttons"] + for button in mouse_buttons["buttonDown"]: + if button not in KeyCode: + continue + self._gui_input._mouse_button_down.add(KeyCode(button)) + for button in mouse_buttons["buttonUp"]: + if button not in KeyCode: + continue + self._gui_input._mouse_button_up.add(KeyCode(button)) + + delta: List[Any] = mouse_json["scrollDelta"] + if len(delta) == 2: + self._gui_input._mouse_scroll_offset += ( + delta[0] if abs(delta[0]) > abs(delta[1]) else delta[1] + ) + # todo: think about ambiguous GuiInput states (key-down and key-up events in the same # frame and other ways that keyHeld, keyDown, and keyUp can be inconsistent. last_client_state = client_states[-1] @@ -180,8 +196,11 @@ def _update_input_state(self, client_states): if "input" in last_client_state else None ) - # TODO: Add mouse support - # mouse_json = last_client_state["mouse"] if "mouse" in last_client_state else None + mouse_json = ( + last_client_state["mouse"] + if "mouse" in last_client_state + else None + ) self._gui_input._key_held.clear() @@ -191,6 +210,13 @@ def _update_input_state(self, client_states): continue self._gui_input._key_held.add(KeyCode(button)) + if mouse_json is not None: + mouse_buttons = mouse_json["buttons"] + for button in mouse_buttons["buttonHeld"]: + if button not in KeyCode: + continue + self._gui_input._mouse_button_held.add(KeyCode(button)) + def debug_visualize_client(self): """Visualize the received VR inputs (head and hands).""" # Sloppy: Use internal debug_line_render to render on server only.