feat: Use W3C actions instead of MJSONWP touch actions and support multi finger touch (iOS)

src/main/java/org/cloud/sonic/driver/ios/IOSDriver.java

@@ -171,7 +171,7 @@ public void unlock() throws SonicRespException {
  * @throws SonicRespException
  */
  public void tap(int x, int y) throws SonicRespException {
- performTouchAction(new TouchActions().press(x, y).release());
+ performTouchAction(new TouchActions.FingerTouchAction().press(x, y).release());
  }
 
  /**
@@ -193,7 +193,7 @@ public void doubleTap(int x, int y) throws SonicRespException {
  * @throws SonicRespException
  */
  public void longPress(int x, int y, int ms) throws SonicRespException {
- performTouchAction(new TouchActions().press(x, y).wait(ms).release());
+ performTouchAction(new TouchActions.FingerTouchAction().press(x, y).wait(ms).release());
  }
 
  /**
@@ -206,7 +206,7 @@ public void longPress(int x, int y, int ms) throws SonicRespException {
  * @throws SonicRespException
  */
  public void swipe(int fromX, int fromY, int toX, int toY) throws SonicRespException {
- performTouchAction(new TouchActions().press(fromX, fromY).wait(300).move(toX, toY).wait(10).release());
+ performTouchAction(new TouchActions.FingerTouchAction().press(fromX, fromY).wait(300).move(toX, toY).wait(10).release());
  }
 
  /**
@@ -230,7 +230,11 @@ public void swipe(double fromX, double fromY, double toX, double toY, double dur
  * @throws SonicRespException
  */
  public void performTouchAction(TouchActions touchActions) throws SonicRespException {
- wdaClient.performTouchAction(touchActions);
+ wdaClient.performTouchAction(touchActions);
+ }
+
+ public void performTouchAction(TouchActions.FingerTouchAction fingerTouchActions) throws SonicRespException {
+ performTouchAction(new TouchActions(fingerTouchActions));
  }
 
  /**

src/main/java/org/cloud/sonic/driver/ios/enums/ActionType.java

@@ -17,10 +17,10 @@
 package org.cloud.sonic.driver.ios.enums;
 
 public enum ActionType {
- PRESS("press"),
- WAIT("wait"),
- MOVE("moveTo"),
- RELEASE("release");
+ PRESS("pointerDown"),
+ WAIT("pause"),
+ MOVE("pointerMove"),
+ RELEASE("pointerUp");
 
  private final String type;
 

src/main/java/org/cloud/sonic/driver/ios/models/TouchActions.java

@@ -16,70 +16,110 @@
  */
 package org.cloud.sonic.driver.ios.models;
 
+import cn.hutool.core.map.MapUtil;
 import lombok.Getter;
 import lombok.ToString;
 import org.cloud.sonic.driver.ios.enums.ActionType;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 @Getter
 @ToString
 public class TouchActions {
 
- private List<TouchAction> actions;
+ private List<FingerTouchAction> actions;
 
  @Getter
  @ToString
- public class TouchAction {
- private String action;
- private Options options;
-
- @Getter
- @ToString
- public class Options {
- private Integer x;
- private Integer y;
- private Integer ms;
+ public static class FingerTouchAction {
+ private final String id;
+ private final String type = "pointer";
+ private final Map<String, String> parameters = MapUtil.of("pointerType", "touch");
+ private final List<TouchAction> actions;
+
+ public FingerTouchAction(String fingerName) {
+ this.id = "finger-" + fingerName;
+ actions = new ArrayList<>();
+ }
+
+ public FingerTouchAction() {
+ this("0");
+ }
+
+ public FingerTouchAction press(int x, int y) {
+ move(x, y);
+ TouchAction touchAction = new TouchAction(ActionType.PRESS);
+ actions.add(touchAction);
+ return this;
+ }
+
+ public FingerTouchAction wait(int ms) {
+ PauseAction touchAction = new PauseAction();
+ touchAction.duration = ms;
+ actions.add(touchAction);
+ return this;
  }
 
+ public FingerTouchAction move(int x, int y) {
+ MoveAction touchAction = new MoveAction();
+ touchAction.x = x;
+ touchAction.y = y;
+ actions.add(touchAction);
+ return this;
+ }
+
+ public FingerTouchAction release() {
+ TouchAction touchAction = new TouchAction(ActionType.RELEASE);
+ actions.add(touchAction);
+ return this;
+ }
+ }
+
+ @Getter
+ @ToString
+ public static class TouchAction {
+ private final String type;
+
  public TouchAction(ActionType actionType) {
- this.action = actionType.getType();
- this.options = new Options();
+ this.type = actionType.getType();
  }
  }
 
- public TouchActions() {
- actions = new ArrayList<>();
+ @Getter
+ @ToString(callSuper = true)
+ public static class MoveAction extends TouchAction {
+ private int x;
+ private int y;
+
+ public MoveAction() {
+ super(ActionType.MOVE);
+ }
  }
 
- public TouchActions press(int x, int y) {
- TouchAction touchAction = new TouchAction(ActionType.PRESS);
- touchAction.options.x = x;
- touchAction.options.y = y;
- actions.add(touchAction);
- return this;
+ @Getter
+ @ToString(callSuper = true)
+ public static class PauseAction extends TouchAction {
+ private int duration;
+
+ public PauseAction() {
+ super(ActionType.WAIT);
+ }
  }
 
- public TouchActions wait(int ms) {
- TouchAction touchAction = new TouchAction(ActionType.WAIT);
- touchAction.options.ms = ms;
- actions.add(touchAction);
- return this;
+ public TouchActions() {
+ actions = new ArrayList<>();
  }
 
- public TouchActions move(int x, int y) {
- TouchAction touchAction = new TouchAction(ActionType.MOVE);
- touchAction.options.x = x;
- touchAction.options.y = y;
- actions.add(touchAction);
- return this;
+ public TouchActions(FingerTouchAction finger) {
+ this();
+ this.actions.add(finger);
  }
 
- public TouchActions release() {
- TouchAction touchAction = new TouchAction(ActionType.RELEASE);
- touchAction.options = null;
- actions.add(touchAction);
- return this;
+ public FingerTouchAction finger(String name) {
+ FingerTouchAction finger = new FingerTouchAction(name);
+ actions.add(finger);
+ return finger;
  }
 }

src/main/java/org/cloud/sonic/driver/ios/service/impl/WdaClientImpl.java

@@ -205,8 +205,8 @@ public void unlock() throws SonicRespException {
  @Override
  public void performTouchAction(TouchActions touchActions) throws SonicRespException {
  checkSessionId();
- BaseResp b = respHandler.getResp(HttpUtil.createPost(remoteUrl + "/session/" + sessionId + "/wda/touch/multi/perform")
- .body(String.valueOf(JSONObject.toJSON(touchActions))));
+ BaseResp b = respHandler.getResp(HttpUtil.createPost(remoteUrl + "/session/" + sessionId + "/actions")
+ .body(JSON.toJSONString(touchActions)));
  if (b.getErr() == null) {
  logger.info("perform action %s.", touchActions.toString());
  } else {

src/test/java/org/cloud/sonic/driver/ios/IOSDriverTest.java

@@ -167,9 +167,26 @@ public void testLongPress() throws SonicRespException {
 
  @Test
  public void testPerformTouchAction() throws SonicRespException, InterruptedException {
- iosDriver.performTouchAction(new TouchActions().press(100, 256).wait(50).move(50, 256).wait(10).release());
+ iosDriver.performTouchAction(new TouchActions.FingerTouchAction().press(100, 256).wait(50).move(50, 256).wait(10).release());
  Thread.sleep(1500);
- iosDriver.performTouchAction(new TouchActions().press(50, 256).wait(50).move(100, 256).wait(10).release());
+ iosDriver.performTouchAction(new TouchActions.FingerTouchAction().press(50, 256).wait(50).move(100, 256).wait(10).release());
+ }
+
+ @Test
+ public void testMultiFingerTouchAction() throws SonicRespException, InterruptedException {
+ iosDriver.appActivate("com.apple.camera");
+ Thread.sleep(1000);
+ TouchActions zoomIn = new TouchActions();
+ zoomIn.finger("0").press(100, 256).wait(50).move(150, 256).wait(10).release();
+ zoomIn.finger("1").press(100, 256).wait(50).move(50, 256).wait(10).release();
+ iosDriver.performTouchAction(zoomIn);
+ Thread.sleep(500);
+ TouchActions zoomOut = new TouchActions();
+ zoomOut.finger("0").press(50, 256).wait(50).move(100, 256).wait(10).release();
+ zoomOut.finger("1").press(150, 256).wait(50).move(100, 256).wait(10).release();
+ iosDriver.performTouchAction(zoomOut);
+ Thread.sleep(500);
+ iosDriver.pressButton(SystemButton.HOME);
  }
 
  @Test
@@ -314,7 +331,7 @@ public void testIsDisplayed() throws SonicRespException {
 
  @Test
  public void testDoubleTap() throws SonicRespException {
- iosDriver.doubleTap(100,100);
+ iosDriver.doubleTap(100, 100);
  iosDriver.pressButton("HOME");
  }