WIP: Add tapping of accessibility element using AXLabel

.gitignore

@@ -51,6 +51,9 @@ fbsimctl/cli-tests/supports_metal
 output/
 .clang-format
 
+# CocoaPods
+Pods/
+
 # Files copied from Xcode
 Fixtures/Source/FBTestRunnerApp/Frameworks
 
@@ -75,6 +78,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+protoc-gen-python_grpc
 
 # Installer logs
 pip-log.txt

idb/cli/commands/hid.py

@@ -7,7 +7,45 @@
 from argparse import ArgumentParser, Namespace
 
 from idb.cli import ClientCommand
-from idb.common.types import Client, HIDButtonType
+from idb.common.types import Client, HIDButtonType, HIDElementType
+
+
+class AXTapCommand(ClientCommand):
+    @property
+    def description(self) -> str:
+        return "Tap Accessibility Element On the Screen"
+
+    @property
+    def name(self) -> str:
+        return "axtap"
+
+    def add_parser_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "element",
+            choices=[element.name for element in HIDElementType],
+            help="Accessibility Type",
+            type=str,
+        )
+        parser.add_argument("label", nargs="?", help="AXLabel", type=str)
+        parser.add_argument(
+            "x", nargs="?", default=1, help="The x-coordinate", type=float
+        )
+        parser.add_argument(
+            "y", nargs="?", default=1, help="The y-coordinate", type=float
+        )
+        parser.add_argument("--duration", help="Press duration", type=float)
+        parser.add_argument("--count", default=1, help="Number of taps", type=int)
+        super().add_parser_arguments(parser)
+
+    async def run_with_client(self, args: Namespace, client: Client) -> None:
+        await client.axtap(
+            element=HIDElementType[args.element],
+            label=args.label,
+            x=args.x,
+            y=args.y,
+            duration=args.duration,
+            count=args.count,
+        )
 
 
 class TapCommand(ClientCommand):
@@ -20,8 +58,8 @@ def name(self) -> str:
         return "tap"
 
     def add_parser_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument("x", help="The x-coordinate", type=int)
-        parser.add_argument("y", help="The y-coordinate", type=int)
+        parser.add_argument("x", help="The x-coordinate", type=float)
+        parser.add_argument("y", help="The y-coordinate", type=float)
         parser.add_argument("--duration", help="Press duration", type=float)
         super().add_parser_arguments(parser)
 

idb/cli/main.py

@@ -54,6 +54,7 @@
 from idb.cli.commands.focus import FocusCommand
 from idb.cli.commands.framework import FrameworkInstallCommand
 from idb.cli.commands.hid import (
+    AXTapCommand,
     ButtonCommand,
     KeyCommand,
     KeySequenceCommand,
@@ -242,6 +243,7 @@ async def gen_main(cmd_input: Optional[List[str]] = None) -> int:
             commands=[
                 AccessibilityInfoAllCommand(),
                 AccessibilityInfoAtPointCommand(),
+                AXTapCommand(),
                 TapCommand(),
                 ButtonCommand(),
                 TextCommand(),

idb/common/types.py

@@ -121,6 +121,13 @@ class HIDButtonType(Enum):
     SIRI = 5
 
 
+class HIDElementType(Enum):
+    back = "back button"
+    button = "button"
+    text = "text"
+    textfield = "text field"
+
+
 ConnectionDestination = Union[str, Address]
 
 

idb/common/udid.py

@@ -10,7 +10,7 @@
 SIMULATOR_UDID = r"^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$"
 OLD_DEVICE_UDID = r"^[0-9a-f]{40}$"
 NEW_DEVICE_UDID = r"^[0-9]{8}-[0-9A-F]{16}$"
-UDID = fr"({SIMULATOR_UDID}|{OLD_DEVICE_UDID}|{NEW_DEVICE_UDID})"
+UDID = rf"({SIMULATOR_UDID}|{OLD_DEVICE_UDID}|{NEW_DEVICE_UDID})"
 
 
 def is_udid(udid: str) -> bool:

idb/grpc/client.py

@@ -8,6 +8,7 @@
 import functools
 import inspect
 import logging
+import json
 import os
 import shutil
 import sys
@@ -65,6 +66,7 @@
     FileEntryInfo,
     FileListing,
     HIDButtonType,
+    HIDElementType,
     HIDEvent,
     IdbConnectionException,
     IdbException,
@@ -789,6 +791,48 @@ async def list_xctests(self) -> List[InstalledTestInfo]:
     async def send_events(self, events: Iterable[HIDEvent]) -> None:
         await self.hid(iterator_to_async_iterator(events))
 
+    @log_and_handle_exceptions
+    async def axtap(
+        self,
+        element: HIDElementType,
+        label: str,
+        x: float,
+        y: float,
+        duration: Optional[float] = None,
+        count: Optional[int] = 1,
+    ) -> None:
+        accessibilityInfo = await self.stub.accessibility_info(
+            AccessibilityInfoRequest(
+                point=None,
+                format=(AccessibilityInfoRequest.LEGACY),
+            )
+        )
+
+        elementFound = False
+
+        for item in json.loads(accessibilityInfo.json):
+            try:
+                axElement = HIDElementType(item["role_description"])
+                axLabel = item["AXLabel"]
+
+                elementFound = axElement == element and (
+                    label is None or axLabel == label
+                )
+
+                if elementFound:
+                    axframe = item["frame"]
+                    x += axframe["x"]
+                    y += axframe["y"]
+                    break
+            except:
+                pass
+
+        if elementFound:
+            for n in range(count):
+                await self.send_events(tap_to_events(x, y, duration))
+        else:
+            print("AXElement with AXLabel: ", label, " type: ", element, " not found.")
+
     @log_and_handle_exceptions
     async def tap(self, x: float, y: float, duration: Optional[float] = None) -> None:
         await self.send_events(tap_to_events(x, y, duration))