From 412c2472a3569ef4f4c17323b5b724d1b8d8c71a Mon Sep 17 00:00:00 2001 From: Matej Knopp Date: Sat, 23 Sep 2023 15:28:53 +0200 Subject: [PATCH 1/2] [macOS] Synchronize modifiers from mouse events for RawKeyboard --- .../Source/FlutterChannelKeyResponder.mm | 67 +++++++++++++++++++ .../Source/FlutterKeyPrimaryResponder.h | 9 +++ .../Source/FlutterKeyboardManager.mm | 7 +- .../Source/FlutterViewControllerTest.mm | 49 ++++++++++++-- 4 files changed, 124 insertions(+), 8 deletions(-) diff --git a/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm b/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm index bcb7aa370ea76..311c6744ef53d 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm @@ -40,6 +40,73 @@ - (nonnull instancetype)initWithChannel:(nonnull FlutterBasicMessageChannel*)cha return self; } +/// Checks single modifier flag from event flags and sends appropriate key event +/// if it is different from the previous state. +- (void)checkModifierFlag:(NSUInteger)modifierFlag + forEventFlags:(NSEventModifierFlags)modifierFlags + keyCode:(NSUInteger)keyCode + timestamp:(NSTimeInterval)timestamp { + if ((modifierFlags & modifierFlag) != (_previouslyPressedFlags & modifierFlag)) { + uint64_t newFlags = (_previouslyPressedFlags & ~modifierFlag) | (modifierFlags & modifierFlag); + + // Sets combined flag if either left or right modifier is pressed, unsets otherwise. + auto updateCombinedFlag = [&](uint64_t side1, uint64_t side2, NSEventModifierFlags flag) { + if (newFlags & (side1 | side2)) { + newFlags |= flag; + } else { + newFlags &= ~flag; + } + }; + updateCombinedFlag(flutter::kModifierFlagShiftLeft, flutter::kModifierFlagShiftRight, + NSEventModifierFlagShift); + updateCombinedFlag(flutter::kModifierFlagControlLeft, flutter::kModifierFlagControlRight, + NSEventModifierFlagControl); + updateCombinedFlag(flutter::kModifierFlagAltLeft, flutter::kModifierFlagAltRight, + NSEventModifierFlagOption); + updateCombinedFlag(flutter::kModifierFlagMetaLeft, flutter::kModifierFlagMetaRight, + NSEventModifierFlagCommand); + + NSEvent* event = [NSEvent keyEventWithType:NSEventTypeFlagsChanged + location:NSZeroPoint + modifierFlags:newFlags + timestamp:timestamp + windowNumber:0 + context:nil + characters:@"" + charactersIgnoringModifiers:@"" + isARepeat:NO + keyCode:keyCode]; + [self handleEvent:event + callback:^(BOOL){ + }]; + }; +} + +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp { + modifierFlags = modifierFlags & ~0x100; + if (_previouslyPressedFlags == modifierFlags) { + return; + } + + [flutter::modifierFlagToKeyCode + enumerateKeysAndObjectsUsingBlock:^(NSNumber* flag, NSNumber* keyCode, BOOL* stop) { + [self checkModifierFlag:[flag unsignedShortValue] + forEventFlags:modifierFlags + keyCode:[keyCode unsignedShortValue] + timestamp:timestamp]; + }]; + + // Caps lock is not included in the modifierFlagToKeyCode map. + [self checkModifierFlag:NSEventModifierFlagCapsLock + forEventFlags:modifierFlags + keyCode:0x00000039 // kVK_CapsLock + timestamp:timestamp]; + + // At the end we should end up with the same modifier flags as the event. + FML_DCHECK(_previouslyPressedFlags == modifierFlags); +} + - (void)handleEvent:(NSEvent*)event callback:(FlutterAsyncKeyCallback)callback { // Remove the modifier bits that Flutter is not interested in. NSEventModifierFlags modifierFlags = event.modifierFlags & ~0x100; diff --git a/shell/platform/darwin/macos/framework/Source/FlutterKeyPrimaryResponder.h b/shell/platform/darwin/macos/framework/Source/FlutterKeyPrimaryResponder.h index b61c790028a3b..2c8918d45d70c 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterKeyPrimaryResponder.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterKeyPrimaryResponder.h @@ -24,6 +24,15 @@ typedef void (^FlutterAsyncKeyCallback)(BOOL handled); @required - (void)handleEvent:(nonnull NSEvent*)event callback:(nonnull FlutterAsyncKeyCallback)callback; +/** + * Synchronize the modifier flags if necessary. The new modifier flag would usually come from mouse + * event and may be out of sync with current keyboard state if the modifier flags have changed while + * window was not key. + */ +@required +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp; + /* A map from macOS key code to logical keyboard. * * The map is assigned on initialization, and updated when the user changes diff --git a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm index 8812df4e948da..00ba4ac38cc39 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm @@ -337,10 +337,9 @@ - (void)buildLayout { - (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags timestamp:(NSTimeInterval)timestamp { - // The embedder responder is the first element in _primaryResponders. - FlutterEmbedderKeyResponder* embedderResponder = - (FlutterEmbedderKeyResponder*)_primaryResponders[0]; - [embedderResponder syncModifiersIfNeeded:modifierFlags timestamp:timestamp]; + for (id responder in _primaryResponders) { + [responder syncModifiersIfNeeded:modifierFlags timestamp:timestamp]; + } } /** diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm index eb28a5b786ad5..7894e9258b9ef 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm @@ -957,13 +957,17 @@ - (bool)testMouseDownUpEventsSentToNextResponder { - (bool)testModifierKeysAreSynthesizedOnMouseMove { id engineMock = flutter::testing::CreateMockFlutterEngine(@""); + id binaryMessengerMock = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [engineMock binaryMessenger]) + .andReturn(binaryMessengerMock); + // Need to return a real renderer to allow view controller to load. FlutterRenderer* renderer_ = [[FlutterRenderer alloc] initWithFlutterEngine:engineMock]; OCMStub([engineMock renderer]).andReturn(renderer_); // Capture calls to sendKeyEvent - __block NSMutableArray* events = - [[NSMutableArray alloc] init]; + __block NSMutableArray* events = [NSMutableArray array]; OCMStub([[engineMock ignoringNonObjectArgs] sendKeyEvent:FlutterKeyEvent {} callback:nil userData:nil]) @@ -973,6 +977,17 @@ - (bool)testModifierKeysAreSynthesizedOnMouseMove { [events addObject:[[KeyEventWrapper alloc] initWithEvent:event]]; })); + __block NSMutableArray* channelEvents = [NSMutableArray array]; + OCMStub([binaryMessengerMock sendOnChannel:@"flutter/keyevent" + message:[OCMArg any] + binaryReply:[OCMArg any]]) + .andDo((^(NSInvocation* invocation) { + NSData* data; + [invocation getArgument:&data atIndex:3]; + id event = [[FlutterJSONMessageCodec sharedInstance] decode:data]; + [channelEvents addObject:event]; + })); + FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engineMock nibName:@"" bundle:nil]; @@ -987,12 +1002,27 @@ - (bool)testModifierKeysAreSynthesizedOnMouseMove { // For each modifier key, check that key events are synthesized. for (NSNumber* keyCode in flutter::keyCodeToModifierFlag) { FlutterKeyEvent* event; + NSDictionary* channelEvent; NSNumber* logicalKey; NSNumber* physicalKey; - NSNumber* flag = flutter::keyCodeToModifierFlag[keyCode]; + NSEventModifierFlags flag = [flutter::keyCodeToModifierFlag[keyCode] unsignedLongValue]; + + // Cocoa event always contain combined flags. + if (flag & (flutter::kModifierFlagShiftLeft | flutter::kModifierFlagShiftRight)) { + flag |= NSEventModifierFlagShift; + } + if (flag & (flutter::kModifierFlagControlLeft | flutter::kModifierFlagControlRight)) { + flag |= NSEventModifierFlagControl; + } + if (flag & (flutter::kModifierFlagAltLeft | flutter::kModifierFlagAltRight)) { + flag |= NSEventModifierFlagOption; + } + if (flag & (flutter::kModifierFlagMetaLeft | flutter::kModifierFlagMetaRight)) { + flag |= NSEventModifierFlagCommand; + } // Should synthesize down event. - NSEvent* mouseEvent = flutter::testing::CreateMouseEvent([flag unsignedLongValue]); + NSEvent* mouseEvent = flutter::testing::CreateMouseEvent(flag); [viewController mouseMoved:mouseEvent]; EXPECT_EQ([events count], 1u); event = events[0].data; @@ -1003,6 +1033,11 @@ - (bool)testModifierKeysAreSynthesizedOnMouseMove { EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); EXPECT_EQ(event->synthesized, true); + channelEvent = channelEvents[0]; + EXPECT_TRUE([channelEvent[@"type"] isEqual:@"keydown"]); + EXPECT_TRUE([channelEvent[@"keyCode"] isEqual:keyCode]); + EXPECT_TRUE([channelEvent[@"modifiers"] isEqual:@(flag)]); + // Should synthesize up event. mouseEvent = flutter::testing::CreateMouseEvent(0x00); [viewController mouseMoved:mouseEvent]; @@ -1015,7 +1050,13 @@ - (bool)testModifierKeysAreSynthesizedOnMouseMove { EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); EXPECT_EQ(event->synthesized, true); + channelEvent = channelEvents[1]; + EXPECT_TRUE([channelEvent[@"type"] isEqual:@"keyup"]); + EXPECT_TRUE([channelEvent[@"keyCode"] isEqual:keyCode]); + EXPECT_TRUE([channelEvent[@"modifiers"] isEqual:@(0)]); + [events removeAllObjects]; + [channelEvents removeAllObjects]; }; return true; From 3d8289b735743053519f906ce9fd0f8e172d293d Mon Sep 17 00:00:00 2001 From: Matej Knopp Date: Wed, 27 Sep 2023 10:49:49 +0200 Subject: [PATCH 2/2] Address nits --- .../macos/framework/Source/FlutterChannelKeyResponder.mm | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm b/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm index 311c6744ef53d..98abd505d4d9b 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterChannelKeyResponder.mm @@ -42,12 +42,13 @@ - (nonnull instancetype)initWithChannel:(nonnull FlutterBasicMessageChannel*)cha /// Checks single modifier flag from event flags and sends appropriate key event /// if it is different from the previous state. -- (void)checkModifierFlag:(NSUInteger)modifierFlag - forEventFlags:(NSEventModifierFlags)modifierFlags +- (void)checkModifierFlag:(NSUInteger)targetMask + forEventFlags:(NSEventModifierFlags)eventFlags keyCode:(NSUInteger)keyCode timestamp:(NSTimeInterval)timestamp { - if ((modifierFlags & modifierFlag) != (_previouslyPressedFlags & modifierFlag)) { - uint64_t newFlags = (_previouslyPressedFlags & ~modifierFlag) | (modifierFlags & modifierFlag); + NSAssert((targetMask & (targetMask - 1)) == 0, @"targetMask must only have one bit set"); + if ((eventFlags & targetMask) != (_previouslyPressedFlags & targetMask)) { + uint64_t newFlags = (_previouslyPressedFlags & ~targetMask) | (eventFlags & targetMask); // Sets combined flag if either left or right modifier is pressed, unsets otherwise. auto updateCombinedFlag = [&](uint64_t side1, uint64_t side2, NSEventModifierFlags flag) {