From 72a9015ad6e8d9c6a970d4bb876cef7cfbfbb5bb Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Sun, 12 Nov 2023 16:06:50 +0100 Subject: [PATCH 01/14] lib/ui: add support for `identifier` in SemanticsNode and SemanticsUpdateBuilder --- lib/ui/semantics.dart | 8 ++++++++ lib/ui/semantics/semantics_node.h | 1 + lib/ui/semantics/semantics_update_builder.cc | 7 +++++++ lib/ui/semantics/semantics_update_builder.h | 1 + 4 files changed, 17 insertions(+) diff --git a/lib/ui/semantics.dart b/lib/ui/semantics.dart index 76c1daa1d5be1..ae922824e17fa 100644 --- a/lib/ui/semantics.dart +++ b/lib/ui/semantics.dart @@ -730,6 +730,9 @@ abstract class SemanticsUpdateBuilder { /// [PlatformDispatcher.onSemanticsActionEvent] callback might be called with /// an action that is no longer possible. /// + /// The `identifier` is a string that describes the node for UI automation. + /// It's not exposed to users. + /// /// The `label` is a string that describes this node. The `value` property /// describes the current value of the node as a string. The `increasedValue` /// string will become the `value` string after a [SemanticsAction.increase] @@ -803,6 +806,7 @@ abstract class SemanticsUpdateBuilder { required double elevation, required double thickness, required Rect rect, + String identifier, required String label, required List labelAttributes, required String value, @@ -872,6 +876,7 @@ base class _NativeSemanticsUpdateBuilder extends NativeFieldWrapperClass1 implem required double elevation, required double thickness, required Rect rect, + String? identifier, required String label, required List labelAttributes, required String value, @@ -910,6 +915,7 @@ base class _NativeSemanticsUpdateBuilder extends NativeFieldWrapperClass1 implem rect.bottom, elevation, thickness, + identifier ?? '', label, labelAttributes, value, @@ -961,6 +967,7 @@ base class _NativeSemanticsUpdateBuilder extends NativeFieldWrapperClass1 implem Handle, Handle, Handle, + Handle, Int32, Handle, Handle, @@ -986,6 +993,7 @@ base class _NativeSemanticsUpdateBuilder extends NativeFieldWrapperClass1 implem double bottom, double elevation, double thickness, + String? identifier, String label, List labelAttributes, String value, diff --git a/lib/ui/semantics/semantics_node.h b/lib/ui/semantics/semantics_node.h index b3789c078c45d..993c1dc80eb40 100644 --- a/lib/ui/semantics/semantics_node.h +++ b/lib/ui/semantics/semantics_node.h @@ -123,6 +123,7 @@ struct SemanticsNode { double scrollExtentMin = std::nan(""); double elevation = 0.0; double thickness = 0.0; + std::string identifier; std::string label; StringAttributes labelAttributes; std::string hint; diff --git a/lib/ui/semantics/semantics_update_builder.cc b/lib/ui/semantics/semantics_update_builder.cc index 2382a901e9aa6..349ba988bcd29 100644 --- a/lib/ui/semantics/semantics_update_builder.cc +++ b/lib/ui/semantics/semantics_update_builder.cc @@ -6,6 +6,7 @@ #include +#include "flutter/fml/logging.h" #include "flutter/lib/ui/floating_point.h" #include "flutter/lib/ui/ui_dart_state.h" #include "third_party/skia/include/core/SkScalar.h" @@ -50,6 +51,7 @@ void SemanticsUpdateBuilder::updateNode( double bottom, double elevation, double thickness, + std::string identifier, std::string label, const std::vector& labelAttributes, std::string value, @@ -88,7 +90,12 @@ void SemanticsUpdateBuilder::updateNode( SafeNarrow(right), SafeNarrow(bottom)); node.elevation = elevation; node.thickness = thickness; + node.identifier = std::move(identifier); node.label = std::move(label); + + FML_LOG(ERROR) << "label: " << node.label + << ", identifier: " << node.identifier; + pushStringAttributes(node.labelAttributes, labelAttributes); node.value = std::move(value); pushStringAttributes(node.valueAttributes, valueAttributes); diff --git a/lib/ui/semantics/semantics_update_builder.h b/lib/ui/semantics/semantics_update_builder.h index 37d45d9f746be..abe4c2806b210 100644 --- a/lib/ui/semantics/semantics_update_builder.h +++ b/lib/ui/semantics/semantics_update_builder.h @@ -49,6 +49,7 @@ class SemanticsUpdateBuilder double bottom, double elevation, double thickness, + std::string identifier, std::string label, const std::vector& labelAttributes, std::string value, From 635567b56570ca60b09086038592378bcdf243ed Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Sun, 12 Nov 2023 16:07:22 +0100 Subject: [PATCH 02/14] PlatformViewAndroidDelegate: update JNI encoding to support `identifier` --- .../platform_view_android_delegate.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc b/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc index aaebecc618c34..1639af639f3d0 100644 --- a/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc +++ b/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate.cc @@ -44,7 +44,7 @@ PlatformViewAndroidDelegate::PlatformViewAndroidDelegate( void PlatformViewAndroidDelegate::UpdateSemantics( const flutter::SemanticsNodeUpdates& update, const flutter::CustomAccessibilityActionUpdates& actions) { - constexpr size_t kBytesPerNode = 47 * sizeof(int32_t); + constexpr size_t kBytesPerNode = 48 * sizeof(int32_t); constexpr size_t kBytesPerChild = sizeof(int32_t); constexpr size_t kBytesPerCustomAction = sizeof(int32_t); constexpr size_t kBytesPerAction = 4 * sizeof(int32_t); @@ -103,6 +103,14 @@ void PlatformViewAndroidDelegate::UpdateSemantics( buffer_float32[position++] = static_cast(node.scrollPosition); buffer_float32[position++] = static_cast(node.scrollExtentMax); buffer_float32[position++] = static_cast(node.scrollExtentMin); + + if (node.identifier.empty()) { + buffer_int32[position++] = -1; + } else { + buffer_int32[position++] = strings.size(); + strings.push_back(node.identifier); + } + if (node.label.empty()) { buffer_int32[position++] = -1; } else { From f4f0328054abfbc7d7ff267dbfa58aeacdec8918 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Sun, 12 Nov 2023 16:13:13 +0100 Subject: [PATCH 03/14] Android AccessibilityBridge: expose `identifier` as `resource-id` --- .../android/io/flutter/view/AccessibilityBridge.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index c2dc77e0034de..dc5c257ade591 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -671,6 +671,9 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { // Work around for https://github.com/flutter/flutter/issues/21030 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { result.setViewIdResourceName(""); + if (semanticsNode.identifier != null) { + result.setViewIdResourceName(semanticsNode.identifier); + } } result.setPackageName(rootAccessibilityView.getContext().getPackageName()); result.setClassName("android.view.View"); @@ -2355,6 +2358,7 @@ private static boolean nullableHasAncestor( private float scrollPosition; private float scrollExtentMax; private float scrollExtentMin; + private String identifier; private String label; private List labelAttributes; private String value; @@ -2487,6 +2491,8 @@ private void log(@NonNull String indent, boolean recursive) { indent + "SemanticsNode id=" + id + + " identifier=" + + identifier + " label=" + label + " actions=" @@ -2550,6 +2556,10 @@ private void updateWith( scrollExtentMin = buffer.getFloat(); int stringIndex = buffer.getInt(); + + identifier = stringIndex == -1 ? null : strings[stringIndex]; + stringIndex = buffer.getInt(); + label = stringIndex == -1 ? null : strings[stringIndex]; labelAttributes = getStringAttributesFromBuffer(buffer, stringAttributeArgs); From 802d0a3bf8dbf5e7d90e7f41c9323eb7b290c756 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 13 Nov 2023 00:28:56 +0100 Subject: [PATCH 04/14] platform_view_android_delegate_unittests: fix tests --- .../platform_view_android_delegate_unittests.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc b/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc index cb3b6f1535331..750ad66414b0b 100644 --- a/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc +++ b/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc @@ -18,11 +18,12 @@ TEST(PlatformViewShell, UpdateSemanticsDoesFlutterViewUpdateSemantics) { flutter::SemanticsNodeUpdates update; flutter::SemanticsNode node0; node0.id = 0; + node0.identifier = "identifier"; node0.label = "label"; node0.tooltip = "tooltip"; update.insert(std::make_pair(0, node0)); - std::vector expected_buffer(188); + std::vector expected_buffer(192); std::vector> expected_string_attribute_args(0); size_t position = 0; int32_t* buffer_int32 = reinterpret_cast(&expected_buffer[0]); @@ -41,6 +42,8 @@ TEST(PlatformViewShell, UpdateSemanticsDoesFlutterViewUpdateSemantics) { buffer_float32[position++] = static_cast(node0.scrollPosition); buffer_float32[position++] = static_cast(node0.scrollExtentMax); buffer_float32[position++] = static_cast(node0.scrollExtentMin); + buffer_int32[position++] = expected_strings.size(); // node0.identifier + expected_strings.push_back(node0.identifier); buffer_int32[position++] = expected_strings.size(); // node0.label expected_strings.push_back(node0.label); buffer_int32[position++] = -1; // node0.labelAttributes @@ -90,13 +93,14 @@ TEST(PlatformViewShell, locale_attribute->type = flutter::StringAttributeType::kLocale; locale_attribute->locale = "en-US"; node0.id = 0; + node0.identifier = "identifier"; node0.label = "label"; node0.labelAttributes.push_back(spell_out_attribute); node0.hint = "hint"; node0.hintAttributes.push_back(locale_attribute); update.insert(std::make_pair(0, node0)); - std::vector expected_buffer(220); + std::vector expected_buffer(224); std::vector> expected_string_attribute_args; size_t position = 0; int32_t* buffer_int32 = reinterpret_cast(&expected_buffer[0]); @@ -115,6 +119,8 @@ TEST(PlatformViewShell, buffer_float32[position++] = static_cast(node0.scrollPosition); buffer_float32[position++] = static_cast(node0.scrollExtentMax); buffer_float32[position++] = static_cast(node0.scrollExtentMin); + buffer_int32[position++] = expected_strings.size(); // node0.identifier + expected_strings.push_back(node0.identifier); buffer_int32[position++] = expected_strings.size(); // node0.label expected_strings.push_back(node0.label); buffer_int32[position++] = 1; // node0.labelAttributes From c9c2ef9896f94570582921fa7d03a5df95e1aedf Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 14 Nov 2023 23:18:15 +0100 Subject: [PATCH 05/14] fix AccessibilityBridgeTest failing --- .../test/io/flutter/view/AccessibilityBridgeTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java b/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java index 12054c9638b5c..9e2f6cb4b0f65 100644 --- a/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java +++ b/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java @@ -2184,6 +2184,7 @@ void addAction(AccessibilityBridge.Action action) { float scrollPosition = 0.0f; float scrollExtentMax = 0.0f; float scrollExtentMin = 0.0f; + String identifier = null; String label = null; List labelAttributes; String value = null; @@ -2241,6 +2242,12 @@ protected void addToBuffer( bytes.putFloat(scrollPosition); bytes.putFloat(scrollExtentMax); bytes.putFloat(scrollExtentMin); + if (identifier == null) { + bytes.putInt(-1); + } else { + strings.add(identifier); + bytes.putInt(strings.size() - 1); + } updateString(label, labelAttributes, bytes, strings, stringAttributeArgs); updateString(value, valueAttributes, bytes, strings, stringAttributeArgs); updateString(increasedValue, increasedValueAttributes, bytes, strings, stringAttributeArgs); From 7b1efadc4360bbdc2a914ef6e5759ea3a0d53130 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Tue, 14 Nov 2023 23:19:03 +0100 Subject: [PATCH 06/14] add new test for identifier in AccessibilityBridgeTest --- .../flutter/view/AccessibilityBridgeTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java b/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java index 9e2f6cb4b0f65..de209e29b448e 100644 --- a/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java +++ b/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java @@ -1103,6 +1103,44 @@ public void itSetsTooltipCorrectly() { assertEquals(actual.toString(), root.tooltip); } + @TargetApi(28) + @Test + public void itSetsIdentifierCorrectly() { + AccessibilityChannel mockChannel = mock(AccessibilityChannel.class); + AccessibilityViewEmbedder mockViewEmbedder = mock(AccessibilityViewEmbedder.class); + AccessibilityManager mockManager = mock(AccessibilityManager.class); + View mockRootView = mock(View.class); + Context context = mock(Context.class); + when(mockRootView.getContext()).thenReturn(context); + when(context.getPackageName()).thenReturn("test"); + AccessibilityBridge accessibilityBridge = + setUpBridge( + /*rootAccessibilityView=*/ mockRootView, + /*accessibilityChannel=*/ mockChannel, + /*accessibilityManager=*/ mockManager, + /*contentResolver=*/ null, + /*accessibilityViewEmbedder=*/ mockViewEmbedder, + /*platformViewsAccessibilityDelegate=*/ null); + + ViewParent mockParent = mock(ViewParent.class); + when(mockRootView.getParent()).thenReturn(mockParent); + when(mockManager.isEnabled()).thenReturn(true); + // Create a node with identifier. + TestSemanticsNode root = new TestSemanticsNode(); + root.id = 0; + root.identifier = "identifier"; + + TestSemanticsUpdate testSemanticsUpdate = root.toUpdate(); + testSemanticsUpdate.sendUpdateToBridge(accessibilityBridge); + + // Test the generated AccessibilityNodeInfo for the node we created and + // verify it has correct identifier (i.e. resource-id per Android + // terminology). + AccessibilityNodeInfo nodeInfo = accessibilityBridge.createAccessibilityNodeInfo(0); + CharSequence actual = nodeInfo.getViewIdResourceName(); + assertEquals(actual.toString(), root.identifier); + } + @Config(sdk = 21) @TargetApi(21) @Test From 5646ef113bd871eba53cef5f42429125e26ad6bf Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Wed, 15 Nov 2023 00:39:47 +0100 Subject: [PATCH 07/14] try to fix api_conform_test --- lib/web_ui/lib/semantics.dart | 2 ++ lib/web_ui/lib/src/engine/semantics/semantics.dart | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/lib/web_ui/lib/semantics.dart b/lib/web_ui/lib/semantics.dart index d1b1904ddaa3a..4ede9f248f485 100644 --- a/lib/web_ui/lib/semantics.dart +++ b/lib/web_ui/lib/semantics.dart @@ -266,6 +266,7 @@ class SemanticsUpdateBuilder { required double elevation, required double thickness, required Rect rect, + required String identifier, required String label, required List labelAttributes, required String value, @@ -300,6 +301,7 @@ class SemanticsUpdateBuilder { scrollExtentMax: scrollExtentMax, scrollExtentMin: scrollExtentMin, rect: rect, + identifier: identifier, label: label, labelAttributes: labelAttributes, value: value, diff --git a/lib/web_ui/lib/src/engine/semantics/semantics.dart b/lib/web_ui/lib/src/engine/semantics/semantics.dart index b27c9233f10a2..a6cfe3703a13f 100644 --- a/lib/web_ui/lib/src/engine/semantics/semantics.dart +++ b/lib/web_ui/lib/src/engine/semantics/semantics.dart @@ -211,6 +211,7 @@ class SemanticsNodeUpdate { required this.scrollExtentMax, required this.scrollExtentMin, required this.rect, + required this.identifier, required this.label, required this.labelAttributes, required this.hint, @@ -273,6 +274,9 @@ class SemanticsNodeUpdate { /// See [ui.SemanticsUpdateBuilder.updateNode]. final ui.Rect rect; + /// See [ui.SemanticsUpdateBuilder.updateNode]. + final String identifier; + /// See [ui.SemanticsUpdateBuilder.updateNode]. final String label; From 88abc9d30c68cf5477fe4b11c20e2e20f06ae228 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Fri, 17 Nov 2023 00:14:53 +0100 Subject: [PATCH 08/14] semantics_update_builder.cc: remove FML_LOG which causes way too many logs --- lib/ui/semantics/semantics_update_builder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ui/semantics/semantics_update_builder.cc b/lib/ui/semantics/semantics_update_builder.cc index 349ba988bcd29..d63ec595704d0 100644 --- a/lib/ui/semantics/semantics_update_builder.cc +++ b/lib/ui/semantics/semantics_update_builder.cc @@ -93,8 +93,8 @@ void SemanticsUpdateBuilder::updateNode( node.identifier = std::move(identifier); node.label = std::move(label); - FML_LOG(ERROR) << "label: " << node.label - << ", identifier: " << node.identifier; + // FML_LOG(ERROR) << "label: " << node.label + // << ", identifier: " << node.identifier; pushStringAttributes(node.labelAttributes, labelAttributes); node.value = std::move(value); From cd98fb246b0b5da5c864843c1eba5a6202a82f60 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 20 Nov 2023 23:30:30 +0100 Subject: [PATCH 09/14] semantics_update_builder: remove debug code --- lib/ui/semantics/semantics_update_builder.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/ui/semantics/semantics_update_builder.cc b/lib/ui/semantics/semantics_update_builder.cc index d63ec595704d0..f7c0d8294fb4f 100644 --- a/lib/ui/semantics/semantics_update_builder.cc +++ b/lib/ui/semantics/semantics_update_builder.cc @@ -6,7 +6,6 @@ #include -#include "flutter/fml/logging.h" #include "flutter/lib/ui/floating_point.h" #include "flutter/lib/ui/ui_dart_state.h" #include "third_party/skia/include/core/SkScalar.h" @@ -92,10 +91,6 @@ void SemanticsUpdateBuilder::updateNode( node.thickness = thickness; node.identifier = std::move(identifier); node.label = std::move(label); - - // FML_LOG(ERROR) << "label: " << node.label - // << ", identifier: " << node.identifier; - pushStringAttributes(node.labelAttributes, labelAttributes); node.value = std::move(value); pushStringAttributes(node.valueAttributes, valueAttributes); From 5a45d2c74a858bbfec55ec3d620ef8d7b6ab5577 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 20 Nov 2023 23:36:50 +0100 Subject: [PATCH 10/14] add `identifier` to missing methods in web UI engine tests --- lib/web_ui/test/engine/semantics/semantics_test.dart | 2 ++ lib/web_ui/test/engine/semantics/semantics_tester.dart | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/web_ui/test/engine/semantics/semantics_test.dart b/lib/web_ui/test/engine/semantics/semantics_test.dart index 29a31b2b870d4..efa17d74c3b41 100644 --- a/lib/web_ui/test/engine/semantics/semantics_test.dart +++ b/lib/web_ui/test/engine/semantics/semantics_test.dart @@ -3069,6 +3069,7 @@ void updateNode( double elevation = 0.0, double thickness = 0.0, ui.Rect rect = ui.Rect.zero, + String identifier = '', String label = '', List labelAttributes = const [], String hint = '', @@ -3109,6 +3110,7 @@ void updateNode( elevation: elevation, thickness: thickness, rect: rect, + identifier: identifier, label: label, labelAttributes: labelAttributes, hint: hint, diff --git a/lib/web_ui/test/engine/semantics/semantics_tester.dart b/lib/web_ui/test/engine/semantics/semantics_tester.dart index 66468f15d775e..88614abd8a04e 100644 --- a/lib/web_ui/test/engine/semantics/semantics_tester.dart +++ b/lib/web_ui/test/engine/semantics/semantics_tester.dart @@ -112,6 +112,7 @@ class SemanticsTester { double? elevation, double? thickness, ui.Rect? rect, + String? identifier, String? label, List? labelAttributes, String? hint, @@ -309,6 +310,7 @@ class SemanticsTester { scrollExtentMax: scrollExtentMax ?? 0, scrollExtentMin: scrollExtentMin ?? 0, rect: effectiveRect, + identifier: identifier ?? '', label: label ?? '', labelAttributes: labelAttributes ?? const [], hint: hint ?? '', From 6a53033b812be00db5ed6b17c9de58350348db73 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 23 Nov 2023 09:42:43 +0100 Subject: [PATCH 11/14] create temporary SemanticsUpdateBuilderNew --- lib/ui/semantics.dart | 340 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 339 insertions(+), 1 deletion(-) diff --git a/lib/ui/semantics.dart b/lib/ui/semantics.dart index ae922824e17fa..bde067498c0a6 100644 --- a/lib/ui/semantics.dart +++ b/lib/ui/semantics.dart @@ -806,7 +806,8 @@ abstract class SemanticsUpdateBuilder { required double elevation, required double thickness, required Rect rect, - String identifier, + // TODO(bartekpacia): Re-add once migration is complete + // String identifier, required String label, required List labelAttributes, required String value, @@ -858,6 +859,343 @@ base class _NativeSemanticsUpdateBuilder extends NativeFieldWrapperClass1 implem @Native(symbol: 'SemanticsUpdateBuilder::Create') external void _constructor(); + @override + void updateNode({ + required int id, + required int flags, + required int actions, + required int maxValueLength, + required int currentValueLength, + required int textSelectionBase, + required int textSelectionExtent, + required int platformViewId, + required int scrollChildren, + required int scrollIndex, + required double scrollPosition, + required double scrollExtentMax, + required double scrollExtentMin, + required double elevation, + required double thickness, + required Rect rect, + // TODO(bartekpacia): Re-add once migration is complete + // String identifier, + required String label, + required List labelAttributes, + required String value, + required List valueAttributes, + required String increasedValue, + required List increasedValueAttributes, + required String decreasedValue, + required List decreasedValueAttributes, + required String hint, + required List hintAttributes, + String? tooltip, + TextDirection? textDirection, + required Float64List transform, + required Int32List childrenInTraversalOrder, + required Int32List childrenInHitTestOrder, + required Int32List additionalActions, + }) { + assert(_matrix4IsValid(transform)); + _updateNode( + id, + flags, + actions, + maxValueLength, + currentValueLength, + textSelectionBase, + textSelectionExtent, + platformViewId, + scrollChildren, + scrollIndex, + scrollPosition, + scrollExtentMax, + scrollExtentMin, + rect.left, + rect.top, + rect.right, + rect.bottom, + elevation, + thickness, + // TODO(bartekpacia): Pass real identifier parameter once migration is complete + '', + label, + labelAttributes, + value, + valueAttributes, + increasedValue, + increasedValueAttributes, + decreasedValue, + decreasedValueAttributes, + hint, + hintAttributes, + tooltip ?? '', + textDirection != null ? textDirection.index + 1 : 0, + transform, + childrenInTraversalOrder, + childrenInHitTestOrder, + additionalActions, + ); + } + @Native< + Void Function( + Pointer, + Int32, + Int32, + Int32, + Int32, + Int32, + Int32, + Int32, + Int32, + Int32, + Int32, + Double, + Double, + Double, + Double, + Double, + Double, + Double, + Double, + Double, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Handle, + Int32, + Handle, + Handle, + Handle, + Handle)>(symbol: 'SemanticsUpdateBuilder::updateNode') + external void _updateNode( + int id, + int flags, + int actions, + int maxValueLength, + int currentValueLength, + int textSelectionBase, + int textSelectionExtent, + int platformViewId, + int scrollChildren, + int scrollIndex, + double scrollPosition, + double scrollExtentMax, + double scrollExtentMin, + double left, + double top, + double right, + double bottom, + double elevation, + double thickness, + String? identifier, + String label, + List labelAttributes, + String value, + List valueAttributes, + String increasedValue, + List increasedValueAttributes, + String decreasedValue, + List decreasedValueAttributes, + String hint, + List hintAttributes, + String tooltip, + int textDirection, + Float64List transform, + Int32List childrenInTraversalOrder, + Int32List childrenInHitTestOrder, + Int32List additionalActions); + + @override + void updateCustomAction({required int id, String? label, String? hint, int overrideId = -1}) { + _updateCustomAction(id, label ?? '', hint ?? '', overrideId); + } + @Native, Int32, Handle, Handle, Int32)>(symbol: 'SemanticsUpdateBuilder::updateCustomAction') + external void _updateCustomAction(int id, String label, String hint, int overrideId); + + @override + SemanticsUpdate build() { + final _NativeSemanticsUpdate semanticsUpdate = _NativeSemanticsUpdate._(); + _build(semanticsUpdate); + return semanticsUpdate; + } + @Native, Handle)>(symbol: 'SemanticsUpdateBuilder::build') + external void _build(_NativeSemanticsUpdate outSemanticsUpdate); +} + +/// An object that creates [SemanticsUpdate] objects. +/// +/// Once created, the [SemanticsUpdate] objects can be passed to +/// [PlatformDispatcher.updateSemantics] to update the semantics conveyed to the +/// user. +@Deprecated('Temporary API. Will be removed once migration is complete.') +abstract class SemanticsUpdateBuilderNew { + /// Creates an empty [SemanticsUpdateBuilderNew] object. + @Deprecated('Temporary API. Will be removed once migration is complete.') + factory SemanticsUpdateBuilderNew() = _NativeSemanticsUpdateBuilderNew; + + /// Update the information associated with the node with the given `id`. + /// + /// The semantics nodes form a tree, with the root of the tree always having + /// an id of zero. The `childrenInTraversalOrder` and `childrenInHitTestOrder` + /// are the ids of the nodes that are immediate children of this node. The + /// former enumerates children in traversal order, and the latter enumerates + /// the same children in the hit test order. The two lists must have the same + /// length and contain the same ids. They may only differ in the order the + /// ids are listed in. For more information about different child orders, see + /// [DebugSemanticsDumpOrder]. + /// + /// The system retains the nodes that are currently reachable from the root. + /// A given update need not contain information for nodes that do not change + /// in the update. If a node is not reachable from the root after an update, + /// the node will be discarded from the tree. + /// + /// The `flags` are a bit field of [SemanticsFlag]s that apply to this node. + /// + /// The `actions` are a bit field of [SemanticsAction]s that can be undertaken + /// by this node. If the user wishes to undertake one of these actions on this + /// node, the [PlatformDispatcher.onSemanticsActionEvent] will be called with + /// a [SemanticsActionEvent] specifying the action to be performed. Because + /// the semantics tree is maintained asynchronously, the + /// [PlatformDispatcher.onSemanticsActionEvent] callback might be called with + /// an action that is no longer possible. + /// + /// The `identifier` is a string that describes the node for UI automation. + /// It's not exposed to users. + /// + /// The `label` is a string that describes this node. The `value` property + /// describes the current value of the node as a string. The `increasedValue` + /// string will become the `value` string after a [SemanticsAction.increase] + /// action is performed. The `decreasedValue` string will become the `value` + /// string after a [SemanticsAction.decrease] action is performed. The `hint` + /// string describes what result an action performed on this node has. The + /// reading direction of all these strings is given by `textDirection`. + /// + /// The `labelAttributes`, `valueAttributes`, `hintAttributes`, + /// `increasedValueAttributes`, and `decreasedValueAttributes` are the lists of + /// [StringAttribute] carried by the `label`, `value`, `hint`, `increasedValue`, + /// and `decreasedValue` respectively. Their contents must not be changed during + /// the semantics update. + /// + /// The `tooltip` is a string that describe additional information when user + /// hover or long press on the backing widget of this semantics node. + /// + /// The fields `textSelectionBase` and `textSelectionExtent` describe the + /// currently selected text within `value`. A value of -1 indicates no + /// current text selection base or extent. + /// + /// The field `maxValueLength` is used to indicate that an editable text + /// field has a limit on the number of characters entered. If it is -1 there + /// is no limit on the number of characters entered. The field + /// `currentValueLength` indicates how much of that limit has already been + /// used up. When `maxValueLength` is >= 0, `currentValueLength` must also be + /// >= 0, otherwise it should be specified to be -1. + /// + /// The field `platformViewId` references the platform view, whose semantics + /// nodes will be added as children to this node. If a platform view is + /// specified, `childrenInHitTestOrder` and `childrenInTraversalOrder` must + /// be empty. A value of -1 indicates that this node is not associated with a + /// platform view. + /// + /// For scrollable nodes `scrollPosition` describes the current scroll + /// position in logical pixel. `scrollExtentMax` and `scrollExtentMin` + /// describe the maximum and minimum in-rage values that `scrollPosition` can + /// be. Both or either may be infinity to indicate unbound scrolling. The + /// value for `scrollPosition` can (temporarily) be outside this range, for + /// example during an overscroll. `scrollChildren` is the count of the + /// total number of child nodes that contribute semantics and `scrollIndex` + /// is the index of the first visible child node that contributes semantics. + /// + /// The `rect` is the region occupied by this node in its own coordinate + /// system. + /// + /// The `transform` is a matrix that maps this node's coordinate system into + /// its parent's coordinate system. + /// + /// The `elevation` describes the distance in z-direction between this node + /// and the `elevation` of the parent. + /// + /// The `thickness` describes how much space this node occupies in the + /// z-direction starting at `elevation`. Basically, in the z-direction the + /// node starts at `elevation` above the parent and ends at `elevation` + + /// `thickness` above the parent. + void updateNode({ + required int id, + required int flags, + required int actions, + required int maxValueLength, + required int currentValueLength, + required int textSelectionBase, + required int textSelectionExtent, + required int platformViewId, + required int scrollChildren, + required int scrollIndex, + required double scrollPosition, + required double scrollExtentMax, + required double scrollExtentMin, + required double elevation, + required double thickness, + required Rect rect, + String identifier, + required String label, + required List labelAttributes, + required String value, + required List valueAttributes, + required String increasedValue, + required List increasedValueAttributes, + required String decreasedValue, + required List decreasedValueAttributes, + required String hint, + required List hintAttributes, + String? tooltip, + TextDirection? textDirection, + required Float64List transform, + required Int32List childrenInTraversalOrder, + required Int32List childrenInHitTestOrder, + required Int32List additionalActions, + }); + + /// Update the custom semantics action associated with the given `id`. + /// + /// The name of the action exposed to the user is the `label`. For overridden + /// standard actions this value is ignored. + /// + /// The `hint` should describe what happens when an action occurs, not the + /// manner in which a tap is accomplished. For example, use "delete" instead + /// of "double tap to delete". + /// + /// The text direction of the `hint` and `label` is the same as the global + /// window. + /// + /// For overridden standard actions, `overrideId` corresponds with a + /// [SemanticsAction.index] value. For custom actions this argument should not be + /// provided. + void updateCustomAction({required int id, String? label, String? hint, int overrideId = -1}); + + /// Creates a [SemanticsUpdate] object that encapsulates the updates recorded + /// by this object. + /// + /// The returned object can be passed to [PlatformDispatcher.updateSemantics] + /// to actually update the semantics retained by the system. + /// + /// This object is unusable after calling build. + SemanticsUpdate build(); +} + +base class _NativeSemanticsUpdateBuilderNew extends NativeFieldWrapperClass1 implements SemanticsUpdateBuilderNew { + _NativeSemanticsUpdateBuilderNew() { _constructor(); } + + @Native(symbol: 'SemanticsUpdateBuilder::Create') + external void _constructor(); + @override void updateNode({ required int id, From 15b7410eb480636ab18200d087b9c379f8d6c74f Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Thu, 23 Nov 2023 17:10:51 +0100 Subject: [PATCH 12/14] web_ui: create temporary SemanticsUpdateBuilderNew --- lib/web_ui/lib/semantics.dart | 100 +++++++++++++++++- .../test/engine/semantics/semantics_test.dart | 3 +- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/lib/web_ui/lib/semantics.dart b/lib/web_ui/lib/semantics.dart index 4ede9f248f485..5ccf9e04664d7 100644 --- a/lib/web_ui/lib/semantics.dart +++ b/lib/web_ui/lib/semantics.dart @@ -245,8 +245,8 @@ class LocaleStringAttribute extends StringAttribute { } } -class SemanticsUpdateBuilder { - SemanticsUpdateBuilder(); +class SemanticsUpdateBuilderNew { + SemanticsUpdateBuilderNew(); final List _nodeUpdates = []; void updateNode({ @@ -339,6 +339,102 @@ class SemanticsUpdateBuilder { } } +class SemanticsUpdateBuilder { + SemanticsUpdateBuilder(); + + final List _nodeUpdates = []; + void updateNode({ + required int id, + required int flags, + required int actions, + required int maxValueLength, + required int currentValueLength, + required int textSelectionBase, + required int textSelectionExtent, + required int platformViewId, + required int scrollChildren, + required int scrollIndex, + required double scrollPosition, + required double scrollExtentMax, + required double scrollExtentMin, + required double elevation, + required double thickness, + required Rect rect, + // TODO(bartekpacia): Re-add once migration is complete + // String identifier, + required String label, + required List labelAttributes, + required String value, + required List valueAttributes, + required String increasedValue, + required List increasedValueAttributes, + required String decreasedValue, + required List decreasedValueAttributes, + required String hint, + required List hintAttributes, + String? tooltip, + TextDirection? textDirection, + required Float64List transform, + required Int32List childrenInTraversalOrder, + required Int32List childrenInHitTestOrder, + required Int32List additionalActions, + }) { + if (transform.length != 16) { + throw ArgumentError('transform argument must have 16 entries.'); + } + _nodeUpdates.add(engine.SemanticsNodeUpdate( + id: id, + flags: flags, + actions: actions, + maxValueLength: maxValueLength, + currentValueLength: currentValueLength, + textSelectionBase: textSelectionBase, + textSelectionExtent: textSelectionExtent, + scrollChildren: scrollChildren, + scrollIndex: scrollIndex, + scrollPosition: scrollPosition, + scrollExtentMax: scrollExtentMax, + scrollExtentMin: scrollExtentMin, + rect: rect, + // TODO(bartekpacia): Pass real identifier parameter once migration is complete + identifier: '', + label: label, + labelAttributes: labelAttributes, + value: value, + valueAttributes: valueAttributes, + increasedValue: increasedValue, + increasedValueAttributes: increasedValueAttributes, + decreasedValue: decreasedValue, + decreasedValueAttributes: decreasedValueAttributes, + hint: hint, + hintAttributes: hintAttributes, + tooltip: tooltip, + textDirection: textDirection, + transform: engine.toMatrix32(transform), + elevation: elevation, + thickness: thickness, + childrenInTraversalOrder: childrenInTraversalOrder, + childrenInHitTestOrder: childrenInHitTestOrder, + additionalActions: additionalActions, + platformViewId: platformViewId, + )); + } + + void updateCustomAction({ + required int id, + String? label, + String? hint, + int overrideId = -1, + }) { + // TODO(yjbanov): implement. + } + SemanticsUpdate build() { + return SemanticsUpdate._( + nodeUpdates: _nodeUpdates, + ); + } +} + abstract class SemanticsUpdate { factory SemanticsUpdate._({List? nodeUpdates}) = engine.SemanticsUpdate; diff --git a/lib/web_ui/test/engine/semantics/semantics_test.dart b/lib/web_ui/test/engine/semantics/semantics_test.dart index efa17d74c3b41..af143f04f7cbd 100644 --- a/lib/web_ui/test/engine/semantics/semantics_test.dart +++ b/lib/web_ui/test/engine/semantics/semantics_test.dart @@ -3110,7 +3110,8 @@ void updateNode( elevation: elevation, thickness: thickness, rect: rect, - identifier: identifier, + // TODO(bartekpacia): Pass real identifier parameter once migration is complete + // identifier: '', label: label, labelAttributes: labelAttributes, hint: hint, From f760cfa5553724d6240242f9a213cd442d657a39 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 27 Nov 2023 19:47:08 +0100 Subject: [PATCH 13/14] semantics.dart: update comment about `identifier` --- lib/ui/semantics.dart | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/ui/semantics.dart b/lib/ui/semantics.dart index bde067498c0a6..ef2a77d440f02 100644 --- a/lib/ui/semantics.dart +++ b/lib/ui/semantics.dart @@ -730,8 +730,9 @@ abstract class SemanticsUpdateBuilder { /// [PlatformDispatcher.onSemanticsActionEvent] callback might be called with /// an action that is no longer possible. /// - /// The `identifier` is a string that describes the node for UI automation. - /// It's not exposed to users. + /// The `identifier` is a string that describes the node for UI automation + /// tools that work by querying the accessibility hierarchy, such as Android + /// UI Automator, iOS XCUITest, or Appium. It's not exposed to users. /// /// The `label` is a string that describes this node. The `value` property /// describes the current value of the node as a string. The `increasedValue` @@ -1068,8 +1069,9 @@ abstract class SemanticsUpdateBuilderNew { /// [PlatformDispatcher.onSemanticsActionEvent] callback might be called with /// an action that is no longer possible. /// - /// The `identifier` is a string that describes the node for UI automation. - /// It's not exposed to users. + /// The `identifier` is a string that describes the node for UI automation + /// tools that work by querying the accessibility hierarchy, such as Android + /// UI Automator, iOS XCUITest, or Appium. It's not exposed to users. /// /// The `label` is a string that describes this node. The `value` property /// describes the current value of the node as a string. The `increasedValue` From c50bceec24433b36860ba82a1388f35fbdd0dbff Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 27 Nov 2023 19:50:05 +0100 Subject: [PATCH 14/14] SemanticsUpdateBuilderNew.updateNode(): make `identifier` parameter required --- lib/ui/semantics.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ui/semantics.dart b/lib/ui/semantics.dart index ef2a77d440f02..7323f64a9ea0a 100644 --- a/lib/ui/semantics.dart +++ b/lib/ui/semantics.dart @@ -1146,7 +1146,7 @@ abstract class SemanticsUpdateBuilderNew { required double elevation, required double thickness, required Rect rect, - String identifier, + required String identifier, required String label, required List labelAttributes, required String value, @@ -1216,7 +1216,7 @@ base class _NativeSemanticsUpdateBuilderNew extends NativeFieldWrapperClass1 imp required double elevation, required double thickness, required Rect rect, - String? identifier, + required String identifier, required String label, required List labelAttributes, required String value, @@ -1255,7 +1255,7 @@ base class _NativeSemanticsUpdateBuilderNew extends NativeFieldWrapperClass1 imp rect.bottom, elevation, thickness, - identifier ?? '', + identifier, label, labelAttributes, value, @@ -1333,7 +1333,7 @@ base class _NativeSemanticsUpdateBuilderNew extends NativeFieldWrapperClass1 imp double bottom, double elevation, double thickness, - String? identifier, + String identifier, String label, List labelAttributes, String value,