Reland "[iOS] Full keyboard access scrolling (#56606)"

lib/ui/semantics.dart

@@ -45,6 +45,7 @@ class SemanticsAction {
   static const int _kMoveCursorBackwardByWordIndex = 1 << 20;
   static const int _kSetTextIndex = 1 << 21;
   static const int _kFocusIndex = 1 << 22;
+  static const int _kScrollToOffsetIndex = 1 << 23;
   // READ THIS: if you add an action here, you MUST update the
   // numSemanticsActions value in testing/dart/semantics_test.dart and
   // lib/web_ui/test/engine/semantics/semantics_api_test.dart, or tests
@@ -86,6 +87,17 @@ class SemanticsAction {
   /// scrollable.
   static const SemanticsAction scrollDown = SemanticsAction._(_kScrollDownIndex, 'scrollDown');
 
+  /// A request to scroll the scrollable container to a given scroll offset.
+  ///
+  /// The payload of this [SemanticsAction] is a flutter-standard-encoded
+  /// [Float64List] of length 2 containing the target horizontal and vertical
+  /// offsets (in logical pixels) the receiving scrollable container should
+  /// scroll to.
+  ///
+  /// This action is used by iOS Full Keyboard Access to reveal contents that
+  /// are currently not visible in the viewport.
+  static const SemanticsAction scrollToOffset = SemanticsAction._(_kScrollToOffsetIndex, 'scrollToOffset');
+
   /// A request to increase the value represented by the semantics node.
   ///
   /// For example, this action might be recognized by a slider control.
@@ -265,6 +277,7 @@ class SemanticsAction {
     _kScrollRightIndex: scrollRight,
     _kScrollUpIndex: scrollUp,
     _kScrollDownIndex: scrollDown,
+    _kScrollToOffsetIndex: scrollToOffset,
     _kIncreaseIndex: increase,
     _kDecreaseIndex: decrease,
     _kShowOnScreenIndex: showOnScreen,
@@ -764,7 +777,7 @@ base class LocaleStringAttribute extends StringAttribute {
     _initLocaleStringAttribute(this, range.start, range.end, locale.toLanguageTag());
   }
 
-  /// The lanuage of this attribute.
+  /// The language of this attribute.
   final Locale locale;
 
   @Native<Void Function(Handle, Int32, Int32, Handle)>(symbol: 'NativeStringAttribute::initLocaleStringAttribute')

lib/ui/semantics/semantics_node.h

@@ -43,6 +43,7 @@ enum class SemanticsAction : int32_t {
   kMoveCursorBackwardByWord = 1 << 20,
   kSetText = 1 << 21,
   kFocus = 1 << 22,
+  kScrollToOffset = 1 << 23,
 };
 
 const int kVerticalScrollSemanticsActions =

lib/web_ui/lib/semantics.dart

@@ -33,13 +33,15 @@ class SemanticsAction {
   static const int _kMoveCursorBackwardByWordIndex = 1 << 20;
   static const int _kSetTextIndex = 1 << 21;
   static const int _kFocusIndex = 1 << 22;
+  static const int _kScrollToOffsetIndex = 1 << 23;
 
   static const SemanticsAction tap = SemanticsAction._(_kTapIndex, 'tap');
   static const SemanticsAction longPress = SemanticsAction._(_kLongPressIndex, 'longPress');
   static const SemanticsAction scrollLeft = SemanticsAction._(_kScrollLeftIndex, 'scrollLeft');
   static const SemanticsAction scrollRight = SemanticsAction._(_kScrollRightIndex, 'scrollRight');
   static const SemanticsAction scrollUp = SemanticsAction._(_kScrollUpIndex, 'scrollUp');
   static const SemanticsAction scrollDown = SemanticsAction._(_kScrollDownIndex, 'scrollDown');
+  static const SemanticsAction scrollToOffset = SemanticsAction._(_kScrollToOffsetIndex, 'scrollToOffset');
   static const SemanticsAction increase = SemanticsAction._(_kIncreaseIndex, 'increase');
   static const SemanticsAction decrease = SemanticsAction._(_kDecreaseIndex, 'decrease');
   static const SemanticsAction showOnScreen = SemanticsAction._(_kShowOnScreenIndex, 'showOnScreen');
@@ -65,6 +67,7 @@ class SemanticsAction {
     _kScrollRightIndex: scrollRight,
     _kScrollUpIndex: scrollUp,
     _kScrollDownIndex: scrollDown,
+    _kScrollToOffsetIndex: scrollToOffset,
     _kIncreaseIndex: increase,
     _kDecreaseIndex: decrease,
     _kShowOnScreenIndex: showOnScreen,

lib/web_ui/test/engine/semantics/semantics_api_test.dart

@@ -29,7 +29,7 @@ void testMain() {
   });
 
   // This must match the number of actions in lib/ui/semantics.dart
-  const int numSemanticsActions = 23;
+  const int numSemanticsActions = 24;
   test('SemanticsAction.values refers to all actions.', () async {
     expect(SemanticsAction.values.length, equals(numSemanticsActions));
     for (int index = 0; index < numSemanticsActions; ++index) {

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -2120,7 +2120,8 @@ public enum Action {
     MOVE_CURSOR_FORWARD_BY_WORD(1 << 19),
     MOVE_CURSOR_BACKWARD_BY_WORD(1 << 20),
     SET_TEXT(1 << 21),
-    FOCUS(1 << 22);
+    FOCUS(1 << 22),
+    SCROLL_TO_OFFSET(1 << 23);
 
     public final int value;
 

shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.h

@@ -18,10 +18,30 @@ NS_ASSUME_NONNULL_BEGIN
  * sends all of selector calls from accessibility services to the
  * owner SemanticsObject.
  */
-@interface FlutterSemanticsScrollView : UIScrollView
+@interface FlutterSemanticsScrollView : UIScrollView <UIScrollViewDelegate>
 
 @property(nonatomic, weak, nullable) SemanticsObject* semanticsObject;
 
+/// Whether this scroll view's content offset is actively being updated by UIKit
+/// or other the system services.
+///
+/// This flag is set by the `FlutterSemanticsScrollView` itself, typically in
+/// one of the `UIScrollViewDelegate` methods.
+///
+/// When this flag is true, the `SemanticsObject` implementation ignores all
+/// content offset updates coming from the Flutter framework, to prevent
+/// potential feedback loops (especially when the framework is only echoing
+/// the new content offset back to this scroll view).
+///
+/// For example, to scroll a scrollable container with iOS full keyboard access,
+/// the iOS focus system uses a display link to scroll the container to the
+/// desired offset animatedly. If the user changes the scroll offset during the
+/// animation, the display link will be invalidated and the scrolling animation
+/// will be interrupted. For simplicity, content offset updates coming from the
+/// framework will be ignored in the relatively short animation duration (~1s),
+/// allowing the scrolling animation to finish.
+@property(nonatomic, readonly) BOOL isDoingSystemScrolling;
+
 - (instancetype)init NS_UNAVAILABLE;
 - (instancetype)initWithFrame:(CGRect)frame NS_UNAVAILABLE;
 - (instancetype)initWithCoder:(NSCoder*)coder NS_UNAVAILABLE;

shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.mm

@@ -15,6 +15,8 @@ - (instancetype)initWithSemanticsObject:(SemanticsObject*)semanticsObject {
   self = [super initWithFrame:CGRectZero];
   if (self) {
     _semanticsObject = semanticsObject;
+    _isDoingSystemScrolling = NO;
+    self.delegate = self;
   }
   return self;
 }
@@ -105,4 +107,14 @@ - (NSInteger)accessibilityElementCount {
   return self.semanticsObject.children.count;
 }
 
+- (void)scrollViewWillEndDragging:(UIScrollView*)scrollView
+                     withVelocity:(CGPoint)velocity
+              targetContentOffset:(inout CGPoint*)targetContentOffset {
+  _isDoingSystemScrolling = YES;
+}
+
+- (void)scrollViewDidEndDecelerating:(UIScrollView*)scrollView {
+  _isDoingSystemScrolling = NO;
+}
+
 @end

shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm

@@ -3,7 +3,10 @@
 // found in the LICENSE file.
 
 #import "SemanticsObject.h"
+#include "flutter/lib/ui/semantics/semantics_node.h"
+#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterCodecs.h"
 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.h"
 
 FLUTTER_ASSERT_ARC
 
@@ -27,10 +30,19 @@
 // translated to calls such as -[NSObject accessibilityActivate]), while most
 // other key events are dispatched to the framework.
 @interface SemanticsObject (UIFocusSystem) <UIFocusItem, UIFocusItemContainer>
+/// The `UIFocusItem` that represents this SemanticsObject.
+///
+///  For regular `SemanticsObject`s, this method returns `self`,
+///  for `FlutterScrollableSemanticsObject`s, this method returns its scroll view.
+- (id<UIFocusItem>)focusItem;
 @end
 
 @implementation SemanticsObject (UIFocusSystem)
 
+- (id<UIFocusItem>)focusItem {
+  return self;
+}
+
 #pragma mark - UIFocusEnvironment Conformance
 
 - (void)setNeedsFocusUpdate {
@@ -49,7 +61,7 @@ - (void)didUpdateFocusInContext:(UIFocusUpdateContext*)context
 
 - (id<UIFocusEnvironment>)parentFocusEnvironment {
   // The root SemanticsObject node's parent is the FlutterView.
-  return self.parent ?: self.bridge->view();
+  return self.parent.focusItem ?: self.bridge->view();
 }
 
 - (NSArray<id<UIFocusEnvironment>>*)preferredFocusEnvironments {
@@ -71,8 +83,57 @@ - (BOOL)canBecomeFocused {
   return self.node.HasAction(flutter::SemanticsAction::kTap);
 }
 
+// The frame is described in the `coordinateSpace` of the
+// `parentFocusEnvironment` (all `parentFocusEnvironment`s are `UIFocusItem`s).
+//
+// See also the `coordinateSpace` implementation.
+// TODO(LongCatIsLooong): use CoreGraphics types.
 - (CGRect)frame {
-  return self.accessibilityFrame;
+  SkPoint quad[4] = {SkPoint::Make(self.node.rect.left(), self.node.rect.top()),
+                     SkPoint::Make(self.node.rect.left(), self.node.rect.bottom()),
+                     SkPoint::Make(self.node.rect.right(), self.node.rect.top()),
+                     SkPoint::Make(self.node.rect.right(), self.node.rect.bottom())};
+
+  SkM44 transform = self.node.transform;
+  FlutterSemanticsScrollView* scrollView;
+  for (SemanticsObject* ancestor = self.parent; ancestor; ancestor = ancestor.parent) {
+    if ([ancestor isKindOfClass:[FlutterScrollableSemanticsObject class]]) {
+      scrollView = ((FlutterScrollableSemanticsObject*)ancestor).scrollView;
+      break;
+    }
+    transform = ancestor.node.transform * transform;
+  }
+
+  for (auto& vertex : quad) {
+    SkV4 vector = transform.map(vertex.x(), vertex.y(), 0, 1);
+    vertex = SkPoint::Make(vector.x / vector.w, vector.y / vector.w);
+  }
+
+  SkRect rect;
+  rect.setBounds(quad, 4);
+  // If this UIFocusItemContainer's coordinateSpace is a UIScrollView, offset
+  // the rect by `contentOffset` because the contentOffset translation is
+  // incorporated into the paint transform at different node depth in UIKit
+  // and Flutter. In Flutter, the translation is added to the cells
+  // while in UIKit the viewport's bounds is manipulated (IOW, each cell's frame
+  // in the UIScrollView coordinateSpace does not change when the UIScrollView
+  // scrolls).
+  CGRect unscaledRect =
+      CGRectMake(rect.x() + scrollView.bounds.origin.x, rect.y() + scrollView.bounds.origin.y,
+                 rect.width(), rect.height());
+  if (scrollView) {
+    return unscaledRect;
+  }
+  // `rect` could be in physical pixels since the root RenderObject ("RenderView")
+  // applies a transform that turns logical pixels to physical pixels. Undo the
+  // transform by dividing the coordinates by the screen's scale factor, if this
+  // UIFocusItem's reported `coordinateSpace` is the root view (which means this
+  // UIFocusItem is not inside of a scroll view).
+  //
+  // Screen can be nil if the FlutterView is covered by another native view.
+  CGFloat scale = (self.bridge->view().window.screen ?: UIScreen.mainScreen).scale;
+  return CGRectMake(unscaledRect.origin.x / scale, unscaledRect.origin.y / scale,
+                    unscaledRect.size.width / scale, unscaledRect.size.height / scale);
 }
 
 #pragma mark - UIFocusItemContainer Conformance
@@ -87,16 +148,94 @@ - (CGRect)frame {
   //
   // This method is only supposed to return items within the given
   // rect but returning everything in the subtree seems to work fine.
-  NSMutableArray<SemanticsObject*>* reversedItems =
+  NSMutableArray<id<UIFocusItem>>* reversedItems =
       [[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count];
   for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) {
-    [reversedItems
-        addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]];
+    SemanticsObject* child = self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i];
+    [reversedItems addObject:child.focusItem];
   }
   return reversedItems;
 }
 
 - (id<UICoordinateSpace>)coordinateSpace {
-  return self.bridge->view();
+  // A regular SemanticsObject uses the same coordinate space as its parent.
+  return self.parent.coordinateSpace ?: self.bridge->view();
+}
+
+@end
+
+/// Scrollable containers interact with the iOS focus engine using the
+/// `UIFocusItemScrollableContainer` protocol. The said protocol (and other focus-related protocols)
+/// does not provide means to inform the focus system of layout changes. In order for the focus
+/// highlight to update properly as the scroll view scrolls, this implementation incorporates a
+/// UIScrollView into the focus hierarchy to workaround the highlight update problem.
+///
+///  As a result, in the current implementation only scrollable containers and the root node
+///  establish their own `coordinateSpace`s. All other `UIFocusItemContainter`s use the same
+///  `coordinateSpace` as the containing UIScrollView, or the root `FlutterView`, whichever is
+///  closer.
+///
+/// See also the `frame` method implementation.
+#pragma mark - Scrolling
+
+@interface FlutterScrollableSemanticsObject (CoordinateSpace)
+@end
+
+@implementation FlutterScrollableSemanticsObject (CoordinateSpace)
+- (id<UICoordinateSpace>)coordinateSpace {
+  // A scrollable SemanticsObject uses the same coordinate space as the scroll view.
+  // This may not work very well in nested scroll views.
+  return self.scrollView;
+}
+
+- (id<UIFocusItem>)focusItem {
+  return self.scrollView;
+}
+
+@end
+
+@interface FlutterSemanticsScrollView (UIFocusItemScrollableContainer) <
+    UIFocusItemScrollableContainer>
+@end
+
+@implementation FlutterSemanticsScrollView (UIFocusItemScrollableContainer)
+
+#pragma mark - FlutterSemanticsScrollView UIFocusItemScrollableContainer Conformance
+
+- (CGSize)visibleSize {
+  return self.frame.size;
+}
+
+- (void)setContentOffset:(CGPoint)contentOffset {
+  [super setContentOffset:contentOffset];
+  // Do no send flutter::SemanticsAction::kScrollToOffset if it's triggered
+  // by a framework update.
+  if (![self.semanticsObject isAccessibilityBridgeAlive] || !self.isDoingSystemScrolling) {
+    return;
+  }
+
+  double offset[2] = {contentOffset.x, contentOffset.y};
+  FlutterStandardTypedData* offsetData = [FlutterStandardTypedData
+      typedDataWithFloat64:[NSData dataWithBytes:&offset length:sizeof(offset)]];
+  NSData* encoded = [[FlutterStandardMessageCodec sharedInstance] encode:offsetData];
+  self.semanticsObject.bridge->DispatchSemanticsAction(
+      self.semanticsObject.uid, flutter::SemanticsAction::kScrollToOffset,
+      fml::MallocMapping::Copy(encoded.bytes, encoded.length));
+}
+
+- (BOOL)canBecomeFocused {
+  return NO;
+}
+
+- (id<UIFocusEnvironment>)parentFocusEnvironment {
+  return self.semanticsObject.parentFocusEnvironment;
+}
+
+- (NSArray<id<UIFocusEnvironment>>*)preferredFocusEnvironments {
+  return nil;
+}
+
+- (NSArray<id<UIFocusItem>>*)focusItemsInRect:(CGRect)rect {
+  return [self.semanticsObject focusItemsInRect:rect];
 }
 @end

shell/platform/darwin/ios/framework/Source/SemanticsObject.h

@@ -10,6 +10,7 @@
 #include "flutter/fml/macros.h"
 #include "flutter/fml/memory/weak_ptr.h"
 #include "flutter/lib/ui/semantics/semantics_node.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge_ios.h"
 
 constexpr int32_t kRootNodeId = 0;
@@ -186,7 +187,7 @@ constexpr float kScrollExtentMaxForInf = 1000;
 /// The semantics object for scrollable. This class creates an UIScrollView to interact with the
 /// iOS.
 @interface FlutterScrollableSemanticsObject : SemanticsObject
-
+@property(nonatomic, readonly) FlutterSemanticsScrollView* scrollView;
 @end
 
 /**

shell/platform/darwin/ios/framework/Source/SemanticsObject.mm

@@ -154,6 +154,8 @@ - (instancetype)initWithBridge:(fml::WeakPtr<flutter::AccessibilityBridgeIos>)br
     _scrollView = [[FlutterSemanticsScrollView alloc] initWithSemanticsObject:self];
     [_scrollView setShowsHorizontalScrollIndicator:NO];
     [_scrollView setShowsVerticalScrollIndicator:NO];
+    [_scrollView setContentInset:UIEdgeInsetsZero];
+    [_scrollView setContentInsetAdjustmentBehavior:UIScrollViewContentInsetAdjustmentNever];
     [self.bridge->view() addSubview:_scrollView];
   }
   return self;
@@ -174,7 +176,10 @@ - (void)accessibilityBridgeDidFinishUpdate {
   // contentOffset is 0.0, only the scroll down action is available.
   self.scrollView.frame = self.accessibilityFrame;
   self.scrollView.contentSize = [self contentSizeInternal];
-  [self.scrollView setContentOffset:[self contentOffsetInternal] animated:NO];
+  // See the documentation on `isDoingSystemScrolling`.
+  if (!self.scrollView.isDoingSystemScrolling) {
+    [self.scrollView setContentOffset:self.contentOffsetInternal animated:NO];
+  }
 }
 
 - (id)nativeAccessibility {