HITL - Add remote mouse input support. (facebookresearch#1870)

* Rename MouseKeyCode to MouseButton.

* Rename MouseKeyCode to MouseButton.

* Formatting fix.

* Review pass.

habitat-hitl/habitat_hitl/_internal/gui_application.py

@@ -7,6 +7,7 @@
 import abc
 import math
 import time
+from typing import List
 
 import magnum as mn
 from magnum.platform.glfw import Application
@@ -32,13 +33,13 @@ def unproject(self, viewport_pos):
 class InputHandlerApplication(Application):
     def __init__(self, config):
         super().__init__(config)
-        self._gui_inputs = []
+        self._gui_inputs: List[GuiInput] = []
 
-    def add_gui_input(self, gui_input):
+    def add_gui_input(self, gui_input: GuiInput) -> None:
         self._gui_inputs.append(gui_input)
 
     def key_press_event(self, event: Application.KeyEvent) -> None:
-        key = MagnumKeyConverter.convert(event.key)
+        key = MagnumKeyConverter.convert_key(event.key)
         if key:
             for wrapper in self._gui_inputs:
                 # If the key is already held, this is a repeat press event and we should
@@ -48,30 +49,30 @@ def key_press_event(self, event: Application.KeyEvent) -> None:
                     wrapper._key_down.add(key)
 
     def key_release_event(self, event: Application.KeyEvent) -> None:
-        key = MagnumKeyConverter.convert(event.key)
+        key = MagnumKeyConverter.convert_key(event.key)
         if key:
             for wrapper in self._gui_inputs:
                 if key in wrapper._key_held:
                     wrapper._key_held.remove(key)
                 wrapper._key_up.add(key)
 
     def mouse_press_event(self, event: Application.MouseEvent) -> None:
-        mouse_button = event.button
-        GuiInput.validate_mouse_button(mouse_button)
-        for wrapper in self._gui_inputs:
-            wrapper._mouse_button_held.add(mouse_button)
-            wrapper._mouse_button_down.add(mouse_button)
+        key = MagnumKeyConverter.convert_mouse_button(event.button)
+        if key:
+            for wrapper in self._gui_inputs:
+                # If the key is already held, this is a repeat press event and we should
+                # ignore it.
+                if key not in wrapper._mouse_button_held:
+                    wrapper._mouse_button_held.add(key)
+                    wrapper._mouse_button_down.add(key)
 
     def mouse_release_event(self, event: Application.MouseEvent) -> None:
-        mouse_button = event.button
-        GuiInput.validate_mouse_button(mouse_button)
-        for wrapper in self._gui_inputs:
-            # In theory, mouse_button should always be present in _mouse_button_held.
-            # In practice, we seem to get spurious release events due to the app
-            # losing focus (e.g. switching to VS code debugger while mouse-clicking)
-            if mouse_button in wrapper._mouse_button_held:
-                wrapper._mouse_button_held.remove(mouse_button)
-                wrapper._mouse_button_up.add(mouse_button)
+        key = MagnumKeyConverter.convert_mouse_button(event.button)
+        if key:
+            for wrapper in self._gui_inputs:
+                if key in wrapper._mouse_button_held:
+                    wrapper._mouse_button_held.remove(key)
+                wrapper._mouse_button_up.add(key)
 
     def mouse_scroll_event(self, event: Application.MouseEvent) -> None:
         # shift+scroll is forced into x direction on mac, seemingly at OS level,

habitat-hitl/habitat_hitl/core/gui_input.py

@@ -4,17 +4,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from habitat_hitl.core.key_mapping import KeyCode
-
-
-class StubNSMeta(type):
-    def __getattr__(cls, name):
-        return None
-
-
-# Stub version of Application.MouseEvent.Button
-class StubMouseNS(metaclass=StubNSMeta):
-    pass
+from habitat_hitl.core.key_mapping import KeyCode, MouseButton
 
 
 class GuiInput:
@@ -25,7 +15,7 @@ class GuiInput:
     """
 
     KeyNS = KeyCode
-    MouseNS = StubMouseNS
+    MouseNS = MouseButton
 
     def __init__(self):
         self._key_held = set()
@@ -37,7 +27,7 @@ def __init__(self):
         self._mouse_button_down = set()
         self._mouse_button_up = set()
         self._relative_mouse_position = [0, 0]
-        self._mouse_scroll_offset = 0
+        self._mouse_scroll_offset = 0.0
         self._mouse_ray = None
 
     def validate_key(key):
@@ -59,9 +49,7 @@ def get_key_up(self, key):
         return key in self._key_up
 
     def validate_mouse_button(mouse_button):
-        # if not do_agnostic_gui_input:
-        #    assert isinstance(mouse_button, Application.MouseEvent.Button)
-        pass
+        assert isinstance(mouse_button, MouseButton)
 
     def get_mouse_button(self, mouse_button):
         GuiInput.validate_mouse_button(mouse_button)
@@ -99,4 +87,4 @@ def on_frame_end(self):
         self._mouse_button_down.clear()
         self._mouse_button_up.clear()
         self._relative_mouse_position = [0, 0]
-        self._mouse_scroll_offset = 0
+        self._mouse_scroll_offset = 0.0

habitat-hitl/habitat_hitl/core/key_mapping.py

@@ -69,6 +69,18 @@ class KeyCode(IntEnum, metaclass=KeyCodeMetaEnum):
     # fmt: on
 
 
+class MouseButton(IntEnum, metaclass=KeyCodeMetaEnum):
+    """
+    Mouse buttons available to control habitat-hitl.
+    """
+
+    # fmt: off
+    LEFT   = 0
+    RIGHT  = 1
+    MIDDLE = 2
+    # fmt: on
+
+
 # On headless systems, we may be unable to import magnum.platform.glfw.Application.
 try:
     from magnum.platform.glfw import Application
@@ -124,9 +136,22 @@ class KeyCode(IntEnum, metaclass=KeyCodeMetaEnum):
         # fmt: on
     }
 
+    magnum_mouse_keymap: Dict[Application.KeyEvent.Key, MouseButton] = {
+        # fmt: off
+        Application.MouseEvent.Button.LEFT   : MouseButton.LEFT  ,
+        Application.MouseEvent.Button.RIGHT  : MouseButton.RIGHT ,
+        Application.MouseEvent.Button.MIDDLE : MouseButton.MIDDLE,
+        # fmt: on
+    }
+
 
 class MagnumKeyConverter:
-    def convert(key: Any) -> Optional[KeyCode]:
+    def convert_key(key: Any) -> Optional[KeyCode]:
         if magnum_enabled and key in magnum_keymap:
             return magnum_keymap[key]
         return None
+
+    def convert_mouse_button(button: Any) -> Optional[MouseButton]:
+        if magnum_enabled and button in magnum_mouse_keymap:
+            return magnum_mouse_keymap[button]
+        return None

habitat-hitl/habitat_hitl/core/remote_client_state.py

@@ -156,10 +156,9 @@ def _update_input_state(self, client_states):
             input_json = (
                 client_state["input"] if "input" in client_state else None
             )
-            # TODO: Add mouse support
-            # mouse_json = (
-            #    client_state["mouse"] if "mouse" in client_state else None
-            # )
+            mouse_json = (
+                client_state["mouse"] if "mouse" in client_state else None
+            )
 
             if input_json is not None:
                 for button in input_json["buttonDown"]:
@@ -171,6 +170,23 @@ def _update_input_state(self, client_states):
                         continue
                     self._gui_input._key_up.add(KeyCode(button))
 
+            if mouse_json is not None:
+                mouse_buttons = mouse_json["buttons"]
+                for button in mouse_buttons["buttonDown"]:
+                    if button not in KeyCode:
+                        continue
+                    self._gui_input._mouse_button_down.add(KeyCode(button))
+                for button in mouse_buttons["buttonUp"]:
+                    if button not in KeyCode:
+                        continue
+                    self._gui_input._mouse_button_up.add(KeyCode(button))
+
+                delta: List[Any] = mouse_json["scrollDelta"]
+                if len(delta) == 2:
+                    self._gui_input._mouse_scroll_offset += (
+                        delta[0] if abs(delta[0]) > abs(delta[1]) else delta[1]
+                    )
+
         # todo: think about ambiguous GuiInput states (key-down and key-up events in the same
         # frame and other ways that keyHeld, keyDown, and keyUp can be inconsistent.
         last_client_state = client_states[-1]
@@ -180,8 +196,11 @@ def _update_input_state(self, client_states):
             if "input" in last_client_state
             else None
         )
-        # TODO: Add mouse support
-        # mouse_json = last_client_state["mouse"] if "mouse" in last_client_state else None
+        mouse_json = (
+            last_client_state["mouse"]
+            if "mouse" in last_client_state
+            else None
+        )
 
         self._gui_input._key_held.clear()
 
@@ -191,6 +210,13 @@ def _update_input_state(self, client_states):
                     continue
                 self._gui_input._key_held.add(KeyCode(button))
 
+        if mouse_json is not None:
+            mouse_buttons = mouse_json["buttons"]
+            for button in mouse_buttons["buttonHeld"]:
+                if button not in KeyCode:
+                    continue
+                self._gui_input._mouse_button_held.add(KeyCode(button))
+
     def debug_visualize_client(self):
         """Visualize the received VR inputs (head and hands)."""
         # Sloppy: Use internal debug_line_render to render on server only.