Handle SurfaceView in a VirtualDisplay (#33599)

Author: Emmanuel Garcia

ci/licenses_golden/licenses_flutter

@@ -1478,6 +1478,8 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/Platfor
 FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java
+FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/SingleViewPresentation.java
+FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/util/PathUtils.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/util/Preconditions.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/util/Predicate.java

shell/platform/android/BUILD.gn

@@ -286,6 +286,8 @@ android_java_sources = [
   "io/flutter/plugin/platform/PlatformViewWrapper.java",
   "io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java",
   "io/flutter/plugin/platform/PlatformViewsController.java",
+  "io/flutter/plugin/platform/SingleViewPresentation.java",
+  "io/flutter/plugin/platform/VirtualDisplayController.java",
   "io/flutter/util/PathUtils.java",
   "io/flutter/util/Preconditions.java",
   "io/flutter/util/Predicate.java",

shell/platform/android/io/flutter/embedding/android/FlutterView.java

@@ -873,6 +873,21 @@ public InputConnection onCreateInputConnection(@NonNull EditorInfo outAttrs) {
     return textInputPlugin.createInputConnection(this, keyboardManager, outAttrs);
   }
 
+  /**
+   * Allows a {@code View} that is not currently the input connection target to invoke commands on
+   * the {@link android.view.inputmethod.InputMethodManager}, which is otherwise disallowed.
+   *
+   * <p>Returns true to allow non-input-connection-targets to invoke methods on {@code
+   * InputMethodManager}, or false to exclusively allow the input connection target to invoke such
+   * methods.
+   */
+  @Override
+  public boolean checkInputConnectionProxy(View view) {
+    return flutterEngine != null
+        ? flutterEngine.getPlatformViewsController().checkInputConnectionProxy(view)
+        : super.checkInputConnectionProxy(view);
+  }
+
   /**
    * Invoked when a hardware key is pressed or released.
    *

shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java

@@ -147,15 +147,18 @@ private void resize(@NonNull MethodCall call, @NonNull MethodChannel.Result resu
                   (double) resizeArgs.get("width"),
                   (double) resizeArgs.get("height"));
           try {
-            final PlatformViewBufferSize sz = handler.resize(resizeRequest);
-            if (sz == null) {
-              result.error("error", "Failed to resize the platform view", null);
-            } else {
-              final Map<String, Object> response = new HashMap<>();
-              response.put("width", (double) sz.width);
-              response.put("height", (double) sz.height);
-              result.success(response);
-            }
+            handler.resize(
+                resizeRequest,
+                (PlatformViewBufferSize bufferSize) -> {
+                  if (bufferSize == null) {
+                    result.error("error", "Failed to resize the platform view", null);
+                  } else {
+                    final Map<String, Object> response = new HashMap<>();
+                    response.put("width", (double) bufferSize.width);
+                    response.put("height", (double) bufferSize.height);
+                    result.success(response);
+                  }
+                });
           } catch (IllegalStateException exception) {
             result.error("error", detailedExceptionString(exception), null);
           }
@@ -298,9 +301,11 @@ public interface PlatformViewsHandler {
      * The Flutter application would like to resize an existing Android {@code View}.
      *
      * @param request The request to resize the platform view.
-     * @return The buffer size where the platform view pixels are written to.
+     * @param onComplete Once the resize is completed, this is the handler to notify the size of the
+     *     platform view buffer.
      */
-    PlatformViewBufferSize resize(@NonNull PlatformViewResizeRequest request);
+    void resize(
+        @NonNull PlatformViewResizeRequest request, @NonNull PlatformViewBufferResized onComplete);
 
     /**
      * The Flutter application would like to change the offset of an existing Android {@code View}.
@@ -418,6 +423,11 @@ public PlatformViewBufferSize(int width, int height) {
     }
   }
 
+  /** Allows to notify when a platform view buffer has been resized. */
+  public interface PlatformViewBufferResized {
+    void run(@Nullable PlatformViewBufferSize bufferSize);
+  }
+
   /** The state of a touch event in Flutter within a platform view. */
   public static class PlatformViewTouch {
     /** The ID of the platform view as seen by the Flutter side. */

shell/platform/android/io/flutter/embedding/engine/systemchannels/TextInputChannel.java

@@ -89,7 +89,9 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result
               try {
                 final JSONObject arguments = (JSONObject) args;
                 final int platformViewId = arguments.getInt("platformViewId");
-                textInputMethodHandler.setPlatformViewClient(platformViewId);
+                final boolean usesVirtualDisplay =
+                    arguments.optBoolean("usesVirtualDisplay", false);
+                textInputMethodHandler.setPlatformViewClient(platformViewId, usesVirtualDisplay);
                 result.success(null);
               } catch (JSONException exception) {
                 result.error("error", exception.getMessage(), null);
@@ -401,8 +403,10 @@ public interface TextInputMethodHandler {
      * different client is set.
      *
      * @param id the ID of the platform view to be set as a text input client.
+     * @param usesVirtualDisplay True if the platform view uses a virtual display, false if it uses
+     *     hybrid composition.
      */
-    void setPlatformViewClient(int id);
+    void setPlatformViewClient(int id, boolean usesVirtualDisplay);
 
     /**
      * Sets the size and the transform matrix of the current text input client.

shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java

@@ -54,6 +54,12 @@ public class TextInputPlugin implements ListenableEditingState.EditingStateWatch
   // Initialize the "last seen" text editing values to a non-null value.
   private TextEditState mLastKnownFrameworkTextEditingState;
 
+  // When true following calls to createInputConnection will return the cached lastInputConnection
+  // if the input
+  // target is a platform view. See the comments on lockPlatformViewInputConnection for more
+  // details.
+  private boolean isInputConnectionLocked;
+
   @SuppressLint("NewApi")
   public TextInputPlugin(
       @NonNull View view,
@@ -99,7 +105,7 @@ public void show() {
 
           @Override
           public void hide() {
-            if (inputTarget.type == InputTarget.Type.PLATFORM_VIEW) {
+            if (inputTarget.type == InputTarget.Type.PHYSICAL_DISPLAY_PLATFORM_VIEW) {
               notifyViewExited();
             } else {
               hideTextInput(mView);
@@ -130,8 +136,8 @@ public void setClient(
           }
 
           @Override
-          public void setPlatformViewClient(int platformViewId) {
-            setPlatformViewTextInputClient(platformViewId);
+          public void setPlatformViewClient(int platformViewId, boolean usesVirtualDisplay) {
+            setPlatformViewTextInputClient(platformViewId, usesVirtualDisplay);
           }
 
           @Override
@@ -176,6 +182,36 @@ ImeSyncDeferringInsetsCallback getImeSyncCallback() {
     return imeSyncCallback;
   }
 
+  /**
+   * Use the current platform view input connection until unlockPlatformViewInputConnection is
+   * called.
+   *
+   * <p>The current input connection instance is cached and any following call to @{link
+   * createInputConnection} returns the cached connection until unlockPlatformViewInputConnection is
+   * called.
+   *
+   * <p>This is a no-op if the current input target isn't a platform view.
+   *
+   * <p>This is used to preserve an input connection when moving a platform view from one virtual
+   * display to another.
+   */
+  public void lockPlatformViewInputConnection() {
+    if (inputTarget.type == InputTarget.Type.VIRTUAL_DISPLAY_PLATFORM_VIEW) {
+      isInputConnectionLocked = true;
+    }
+  }
+
+  /**
+   * Unlocks the input connection.
+   *
+   * <p>See also: @{link lockPlatformViewInputConnection}.
+   */
+  public void unlockPlatformViewInputConnection() {
+    if (inputTarget.type == InputTarget.Type.VIRTUAL_DISPLAY_PLATFORM_VIEW) {
+      isInputConnectionLocked = false;
+    }
+  }
+
   /**
    * Detaches the text input plugin from the platform views controller.
    *
@@ -259,10 +295,21 @@ public InputConnection createInputConnection(
       return null;
     }
 
-    if (inputTarget.type == InputTarget.Type.PLATFORM_VIEW) {
+    if (inputTarget.type == InputTarget.Type.PHYSICAL_DISPLAY_PLATFORM_VIEW) {
       return null;
     }
 
+    if (inputTarget.type == InputTarget.Type.VIRTUAL_DISPLAY_PLATFORM_VIEW) {
+      if (isInputConnectionLocked) {
+        return lastInputConnection;
+      }
+      lastInputConnection =
+          platformViewsController
+              .getPlatformViewById(inputTarget.id)
+              .onCreateInputConnection(outAttrs);
+      return lastInputConnection;
+    }
+
     outAttrs.inputType =
         inputTypeFromTextInputType(
             configuration.inputType,
@@ -317,7 +364,9 @@ public InputConnection getLastInputConnection() {
    * input connection.
    */
   public void clearPlatformViewClient(int platformViewId) {
-    if (inputTarget.type == InputTarget.Type.PLATFORM_VIEW && inputTarget.id == platformViewId) {
+    if ((inputTarget.type == InputTarget.Type.VIRTUAL_DISPLAY_PLATFORM_VIEW
+            || inputTarget.type == InputTarget.Type.PHYSICAL_DISPLAY_PLATFORM_VIEW)
+        && inputTarget.id == platformViewId) {
       inputTarget = new InputTarget(InputTarget.Type.NO_TARGET, 0);
       notifyViewExited();
       mImm.hideSoftInputFromWindow(mView.getApplicationWindowToken(), 0);
@@ -378,13 +427,26 @@ void setTextInputClient(int client, TextInputChannel.Configuration configuration
     // setTextInputClient will be followed by a call to setTextInputEditingState.
     // Do a restartInput at that time.
     mRestartInputPending = true;
+    unlockPlatformViewInputConnection();
     lastClientRect = null;
     mEditable.addEditingStateListener(this);
   }
 
-  private void setPlatformViewTextInputClient(int platformViewId) {
-    inputTarget = new InputTarget(InputTarget.Type.PLATFORM_VIEW, platformViewId);
-    lastInputConnection = null;
+  private void setPlatformViewTextInputClient(int platformViewId, boolean usesVirtualDisplay) {
+    if (usesVirtualDisplay) {
+      // We need to make sure that the Flutter view is focused so that no imm operations get short
+      // circuited.
+      // Not asking for focus here specifically manifested in a bug on API 28 devices where the
+      // platform view's request to show a keyboard was ignored.
+      mView.requestFocus();
+      inputTarget = new InputTarget(InputTarget.Type.VIRTUAL_DISPLAY_PLATFORM_VIEW, platformViewId);
+      mImm.restartInput(mView);
+      mRestartInputPending = false;
+    } else {
+      inputTarget =
+          new InputTarget(InputTarget.Type.PHYSICAL_DISPLAY_PLATFORM_VIEW, platformViewId);
+      lastInputConnection = null;
+    }
   }
 
   private static boolean composingChanged(
@@ -475,10 +537,28 @@ public void inspect(double x, double y) {
 
   @VisibleForTesting
   void clearTextInputClient() {
+    if (inputTarget.type == InputTarget.Type.VIRTUAL_DISPLAY_PLATFORM_VIEW) {
+      // This only applies to platform views that use a virtual display.
+      // Focus changes in the framework tree have no guarantees on the order focus nodes are
+      // notified. A node that lost focus may be notified before or after a node that gained focus.
+      // When moving the focus from a Flutter text field to an AndroidView, it is possible that the
+      // Flutter text field's focus node will be notified that it lost focus after the AndroidView
+      // was notified that it gained focus. When this happens the text field will send a
+      // clearTextInput command which we ignore.
+      // By doing this we prevent the framework from clearing a platform view input client (the only
+      // way to do so is to set a new framework text client). I don't see an obvious use case for
+      // "clearing" a platform view's text input client, and it may be error prone as we don't know
+      // how the platform view manages the input connection and we probably shouldn't interfere.
+      // If we ever want to allow the framework to clear a platform view text client we should
+      // probably consider changing the focus manager such that focus nodes that lost focus are
+      // notified before focus nodes that gained focus as part of the same focus event.
+      return;
+    }
     mEditable.removeEditingStateListener(this);
     notifyViewExited();
     updateAutofillConfigurationIfNeeded(null);
     inputTarget = new InputTarget(InputTarget.Type.NO_TARGET, 0);
+    unlockPlatformViewInputConnection();
     lastClientRect = null;
   }
 
@@ -488,9 +568,12 @@ enum Type {
       // InputConnection is managed by the TextInputPlugin, and events are forwarded to the Flutter
       // framework.
       FRAMEWORK_CLIENT,
-      // InputConnection is managed by a platform view that is embeded in the Android view
-      // hierarchy.
-      PLATFORM_VIEW,
+      // InputConnection is managed by a platform view that is presented on a virtual display.
+      VIRTUAL_DISPLAY_PLATFORM_VIEW,
+      // InputConnection is managed by a platform view that is embedded in the activity's view
+      // hierarchy. This view hierarchy is displayed in a physical display within the aplication
+      // display area.
+      PHYSICAL_DISPLAY_PLATFORM_VIEW,
     }
 
     public InputTarget(@NonNull Type type, int id) {

shell/platform/android/io/flutter/plugin/platform/PlatformView.java

@@ -66,22 +66,16 @@ default void onFlutterViewDetached() {}
    *
    * <p>This hook only exists for rare cases where the plugin relies on the state of the input
    * connection. This probably doesn't need to be implemented.
-   *
-   * <p>This method is deprecated, and will be removed in a future release.
    */
   @SuppressLint("NewApi")
-  @Deprecated
   default void onInputConnectionLocked() {}
 
   /**
    * Callback fired when the platform input connection has been unlocked.
    *
    * <p>This hook only exists for rare cases where the plugin relies on the state of the input
    * connection. This probably doesn't need to be implemented.
-   *
-   * <p>This method is deprecated, and will be removed in a future release.
    */
   @SuppressLint("NewApi")
-  @Deprecated
   default void onInputConnectionUnlocked() {}
 }

shell/platform/android/io/flutter/plugin/platform/PlatformViewFactory.java

@@ -28,7 +28,7 @@ public PlatformViewFactory(@Nullable MessageCodec<Object> createArgsCodec) {
    *     null, or no arguments were sent from the Flutter app.
    */
   @NonNull
-  public abstract PlatformView create(@Nullable Context context, int viewId, @Nullable Object args);
+  public abstract PlatformView create(Context context, int viewId, @Nullable Object args);
 
   /** Returns the codec to be used for decoding the args parameter of {@link #create}. */
   @Nullable

shell/platform/android/io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java

@@ -18,6 +18,9 @@ public interface PlatformViewsAccessibilityDelegate {
   @Nullable
   View getPlatformViewById(int viewId);
 
+  /** Returns true if the platform view uses virtual displays. */
+  boolean usesVirtualDisplay(int id);
+
   /**
    * Attaches an accessibility bridge for this platform views accessibility delegate.
    *