From 954ed169bb64f53754aff7b9f3e71d256d79ec37 Mon Sep 17 00:00:00 2001 From: js_john Date: Sat, 13 Mar 2021 18:02:02 +0800 Subject: [PATCH 1/2] iOS: use GCMouse API on iOS 14 and above --- .../VMDisplayMetalViewController+Pointer.h | 3 +- .../VMDisplayMetalViewController+Pointer.m | 58 +++++++++++++++++++ .../Display/VMDisplayMetalViewController.m | 1 + 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.h b/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.h index 16a0c943b..842bb06a3 100644 --- a/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.h +++ b/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.h @@ -16,7 +16,7 @@ #import "UIKit/UIKit.h" #import "VMDisplayMetalViewController.h" - +@import GameController; NS_ASSUME_NONNULL_BEGIN NS_AVAILABLE_IOS(13.4) @@ -25,6 +25,7 @@ NS_AVAILABLE_IOS(13.4) @property (nonatomic, readonly) BOOL hasTouchpadPointer; -(void)initPointerInteraction; +-(void)initGCMouse; @end diff --git a/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m b/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m index ad5b09e16..10cc32d71 100644 --- a/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m +++ b/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m @@ -21,6 +21,10 @@ #import "VMDisplayMetalViewController+Pointer.h" #import "VMCursor.h" #import "CSDisplayMetal.h" +#import "VMScroll.h" +#import "UTMVirtualMachine.h" +#import "UTMVirtualMachine+SPICE.h" +#import "UTMLogging.h" NS_AVAILABLE_IOS(13.4) @implementation VMDisplayMetalViewController (Pointer) @@ -38,6 +42,55 @@ -(void)initPointerInteraction { } } +- (BOOL)switchMouseType:(VMMouseType)type { + BOOL shouldUseServerMouse = YES; + self.vmInput.inhibitCursor = NO; + if (shouldUseServerMouse != self.vmInput.serverModeCursor) { + UTMLog(@"Switching mouse mode to server:%d for type:%ld", shouldUseServerMouse, type); + [self.vm requestInputTablet:!shouldUseServerMouse]; + return YES; + } + return NO; +} + +- (void) initGCMouse { + if (@available(iOS 14.0, *)) { //if ios 14.0 above, use CGMouse instead + GCMouse *mouse = GCMouse.current; + __weak typeof(self) _self = self; + VMDisplayMetalViewController *s = _self; + mouse.mouseInput.mouseMovedHandler = ^(GCMouseInput * _Nonnull mouse, float deltaX, float deltaY) { + [self switchMouseType:VMMouseTypeRelative]; + [self.vmInput sendMouseMotion:self.mouseButtonDown point:CGPointMake(deltaX, -deltaY)]; + }; + mouse.mouseInput.leftButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { + s->_mouseLeftDown = pressed; + [s.vmInput sendMouseButton:kCSInputButtonLeft pressed:pressed point:CGPointZero]; + }; + + mouse.mouseInput.rightButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { + s->_mouseRightDown = pressed; + [s.vmInput sendMouseButton:kCSInputButtonRight pressed:pressed point:CGPointZero]; + + }; + + mouse.mouseInput.middleButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { + s->_mouseMiddleDown = pressed; + [s.vmInput sendMouseButton:kCSInputButtonMiddle pressed:pressed point:CGPointZero]; + + }; + + // no handler to the gcmouse scroll event, gestureScroll works fine. + } +} + +- (BOOL) prefersPointerLocked { + if (self.indirectMouseType == VMMouseTypeAbsolute || self.indirectMouseType == VMMouseTypeAbsoluteHideCursor) { + return YES; + } else { + return NO; + } +} + - (BOOL)hasTouchpadPointer { return !self.vmConfiguration.inputLegacy && !self.vmInput.serverModeCursor && self.indirectMouseType != VMMouseTypeRelative; } @@ -74,6 +127,11 @@ - (bool)isPointOnVMDisplay:(CGPoint)pos { - (UIPointerRegion *)pointerInteraction:(UIPointerInteraction *)interaction regionForRequest:(UIPointerRegionRequest *)request defaultRegion:(UIPointerRegion *)defaultRegion { + if (@available(iOS 14.0, *)) { + if (self.prefersPointerLocked) { + return nil; + } + } // Requesting region for the VM display? if (interaction.view == self.mtkView && self.hasTouchpadPointer) { // Then we need to find out if the pointer is in the actual display area or outside diff --git a/Platform/iOS/Display/VMDisplayMetalViewController.m b/Platform/iOS/Display/VMDisplayMetalViewController.m index 7589847c4..12147172a 100644 --- a/Platform/iOS/Display/VMDisplayMetalViewController.m +++ b/Platform/iOS/Display/VMDisplayMetalViewController.m @@ -73,6 +73,7 @@ - (void)viewDidLoad { [self initTouch]; [self initGamepad]; + [self initGCMouse]; // Pointing device support on iPadOS 13.4 GM or later if (@available(iOS 13.4, *)) { // Betas of iPadOS 13.4 did not include this API, that's why I check if the class exists From 05a72f51fae9e2bdbf3825793128538f9b479913 Mon Sep 17 00:00:00 2001 From: osy <50960678+osy@users.noreply.github.com> Date: Sun, 25 Apr 2021 18:26:40 -0700 Subject: [PATCH 2/2] iOS: support multiple GCMouse and iOS Simulator Multiple mice could be connected. We will always use the most "recent" mice. Also ignore UITouchTypeIndirectPointer events when mouse is captured. Fixes #2363 --- .../VMDisplayMetalViewController+Pointer.m | 115 ++++++++++-------- .../VMDisplayMetalViewController+Touch.m | 7 +- .../Display/VMDisplayMetalViewController.h | 1 + 3 files changed, 72 insertions(+), 51 deletions(-) diff --git a/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m b/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m index 10cc32d71..320597786 100644 --- a/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m +++ b/Platform/iOS/Display/VMDisplayMetalViewController+Pointer.m @@ -26,9 +26,74 @@ #import "UTMVirtualMachine+SPICE.h" #import "UTMLogging.h" +@interface VMDisplayMetalViewController () + +- (BOOL)switchMouseType:(VMMouseType)type; // defined in VMDisplayMetalViewController+Touch.m + +@end + NS_AVAILABLE_IOS(13.4) @implementation VMDisplayMetalViewController (Pointer) +#pragma mark - GCMouse + +- (void)initGCMouse { + if (@available(iOS 14.0, *)) { //if ios 14.0 above, use CGMouse instead + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(mouseDidBecomeCurrent:) name:GCMouseDidBecomeCurrentNotification object:nil]; + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(mouseDidStopBeingCurrent:) name:GCMouseDidStopBeingCurrentNotification object:nil]; + } +} + +- (BOOL)prefersPointerLocked { + return _mouseCaptured; +} + +- (void)mouseDidBecomeCurrent:(NSNotification *)notification API_AVAILABLE(ios(14)) { + GCMouse *mouse = notification.object; + UTMLog(@"mouseDidBecomeCurrent: %p", mouse); + if (!mouse) { + UTMLog(@"invalid mouse object!"); + return; + } + mouse.mouseInput.mouseMovedHandler = ^(GCMouseInput * _Nonnull mouse, float deltaX, float deltaY) { + [self.vmInput sendMouseMotion:self.mouseButtonDown point:CGPointMake(deltaX, -deltaY)]; + }; + mouse.mouseInput.leftButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { + self->_mouseLeftDown = pressed; + [self.vmInput sendMouseButton:kCSInputButtonLeft pressed:pressed point:CGPointZero]; + }; + mouse.mouseInput.rightButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { + self->_mouseRightDown = pressed; + [self.vmInput sendMouseButton:kCSInputButtonRight pressed:pressed point:CGPointZero]; + + }; + mouse.mouseInput.middleButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { + self->_mouseMiddleDown = pressed; + [self.vmInput sendMouseButton:kCSInputButtonMiddle pressed:pressed point:CGPointZero]; + }; + // no handler to the gcmouse scroll event, gestureScroll works fine. + [self switchMouseType:VMMouseTypeRelative]; + _mouseCaptured = YES; + dispatch_async(dispatch_get_main_queue(), ^{ + [self setNeedsUpdateOfPrefersPointerLocked]; + }); +} + +- (void)mouseDidStopBeingCurrent:(NSNotification *)notification API_AVAILABLE(ios(14)) { + GCMouse *mouse = notification.object; + UTMLog(@"mouseDidStopBeingCurrent: %p", mouse); + mouse.mouseInput.mouseMovedHandler = nil; + mouse.mouseInput.leftButton.pressedChangedHandler = nil; + mouse.mouseInput.rightButton.pressedChangedHandler = nil; + mouse.mouseInput.middleButton.pressedChangedHandler = nil; + _mouseCaptured = NO; + dispatch_async(dispatch_get_main_queue(), ^{ + [self setNeedsUpdateOfPrefersPointerLocked]; + }); +} + +#pragma mark - UIPointerInteractionDelegate + // Add pointer interaction to VM view -(void)initPointerInteraction { [self.mtkView addInteraction:[[UIPointerInteraction alloc] initWithDelegate:self]]; @@ -42,60 +107,10 @@ -(void)initPointerInteraction { } } -- (BOOL)switchMouseType:(VMMouseType)type { - BOOL shouldUseServerMouse = YES; - self.vmInput.inhibitCursor = NO; - if (shouldUseServerMouse != self.vmInput.serverModeCursor) { - UTMLog(@"Switching mouse mode to server:%d for type:%ld", shouldUseServerMouse, type); - [self.vm requestInputTablet:!shouldUseServerMouse]; - return YES; - } - return NO; -} - -- (void) initGCMouse { - if (@available(iOS 14.0, *)) { //if ios 14.0 above, use CGMouse instead - GCMouse *mouse = GCMouse.current; - __weak typeof(self) _self = self; - VMDisplayMetalViewController *s = _self; - mouse.mouseInput.mouseMovedHandler = ^(GCMouseInput * _Nonnull mouse, float deltaX, float deltaY) { - [self switchMouseType:VMMouseTypeRelative]; - [self.vmInput sendMouseMotion:self.mouseButtonDown point:CGPointMake(deltaX, -deltaY)]; - }; - mouse.mouseInput.leftButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { - s->_mouseLeftDown = pressed; - [s.vmInput sendMouseButton:kCSInputButtonLeft pressed:pressed point:CGPointZero]; - }; - - mouse.mouseInput.rightButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { - s->_mouseRightDown = pressed; - [s.vmInput sendMouseButton:kCSInputButtonRight pressed:pressed point:CGPointZero]; - - }; - - mouse.mouseInput.middleButton.pressedChangedHandler = ^(GCControllerButtonInput * _Nonnull button, float value, BOOL pressed) { - s->_mouseMiddleDown = pressed; - [s.vmInput sendMouseButton:kCSInputButtonMiddle pressed:pressed point:CGPointZero]; - - }; - - // no handler to the gcmouse scroll event, gestureScroll works fine. - } -} - -- (BOOL) prefersPointerLocked { - if (self.indirectMouseType == VMMouseTypeAbsolute || self.indirectMouseType == VMMouseTypeAbsoluteHideCursor) { - return YES; - } else { - return NO; - } -} - - (BOOL)hasTouchpadPointer { return !self.vmConfiguration.inputLegacy && !self.vmInput.serverModeCursor && self.indirectMouseType != VMMouseTypeRelative; } -#pragma mark - UIPointerInteractionDelegate - (UIPointerStyle *)pointerInteraction:(UIPointerInteraction *)interaction styleForRegion:(UIPointerRegion *)region { // Hide cursor while hovering in VM view if (interaction.view == self.mtkView && self.hasTouchpadPointer) { diff --git a/Platform/iOS/Display/VMDisplayMetalViewController+Touch.m b/Platform/iOS/Display/VMDisplayMetalViewController+Touch.m index fc295f582..945229c23 100644 --- a/Platform/iOS/Display/VMDisplayMetalViewController+Touch.m +++ b/Platform/iOS/Display/VMDisplayMetalViewController+Touch.m @@ -583,8 +583,13 @@ - (BOOL)switchMouseType:(VMMouseType)type { #pragma mark - Touch event handling - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event { - if (!self.vmConfiguration.inputLegacy) { + if (!_mouseCaptured && !self.vmConfiguration.inputLegacy) { for (UITouch *touch in [event touchesForView:self.mtkView]) { + if (@available(iOS 14, *)) { + if (self.prefersPointerLocked && (touch.type == UITouchTypeIndirect || touch.type == UITouchTypeIndirectPointer)) { + continue; // skip indirect touches if we are capturing mouse input + } + } VMMouseType type = [self touchTypeToMouseType:touch.type]; if ([self switchMouseType:type]) { [self dragCursor:UIGestureRecognizerStateEnded primary:YES secondary:YES middle:YES]; // reset drag diff --git a/Platform/iOS/Display/VMDisplayMetalViewController.h b/Platform/iOS/Display/VMDisplayMetalViewController.h index 23c1191b4..7380c7ce8 100644 --- a/Platform/iOS/Display/VMDisplayMetalViewController.h +++ b/Platform/iOS/Display/VMDisplayMetalViewController.h @@ -37,6 +37,7 @@ NS_ASSUME_NONNULL_BEGIN BOOL _pencilForceRightClickOnce; VMCursor *_cursor; VMScroll *_scroll; + BOOL _mouseCaptured; // Gestures UISwipeGestureRecognizer *_swipeUp;