Include TextureLayer in contents of toImageSync (#35608)

Author: Jonah Williams

flow/layers/layer_tree.cc

@@ -171,29 +171,33 @@ void LayerTree::Paint(CompositorContext::ScopedFrame& frame,
   }
 }
 
-sk_sp<DisplayList> LayerTree::Flatten(const SkRect& bounds) {
+sk_sp<DisplayList> LayerTree::Flatten(
+    const SkRect& bounds,
+    std::shared_ptr<TextureRegistry> texture_registry,
+    GrDirectContext* gr_context) {
   TRACE_EVENT0("flutter", "LayerTree::Flatten");
 
   DisplayListCanvasRecorder builder(bounds);
 
   MutatorsStack unused_stack;
   const FixedRefreshRateStopwatch unused_stopwatch;
   SkMatrix root_surface_transformation;
+
   // No root surface transformation. So assume identity.
   root_surface_transformation.reset();
 
   PrerollContext preroll_context{
       // clang-format off
       .raster_cache                  = nullptr,
-      .gr_context                    = nullptr,
+      .gr_context                    = gr_context,
       .view_embedder                 = nullptr,
       .mutators_stack                = unused_stack,
       .dst_color_space               = nullptr,
       .cull_rect                     = kGiantRect,
       .surface_needs_readback        = false,
       .raster_time                   = unused_stopwatch,
       .ui_time                       = unused_stopwatch,
-      .texture_registry              = nullptr,
+      .texture_registry              = texture_registry,
       .checkerboard_offscreen_layers = false,
       .frame_device_pixel_ratio      = device_pixel_ratio_
       // clang-format on
@@ -209,12 +213,12 @@ sk_sp<DisplayList> LayerTree::Flatten(const SkRect& bounds) {
       // clang-format off
       .internal_nodes_canvas         = &internal_nodes_canvas,
       .leaf_nodes_canvas             = &builder,
-      .gr_context                    = nullptr,
+      .gr_context                    = gr_context,
       .dst_color_space               = nullptr,
       .view_embedder                 = nullptr,
       .raster_time                   = unused_stopwatch,
       .ui_time                       = unused_stopwatch,
-      .texture_registry              = nullptr,
+      .texture_registry              = texture_registry,
       .raster_cache                  = nullptr,
       .checkerboard_offscreen_layers = false,
       .frame_device_pixel_ratio      = device_pixel_ratio_,

flow/layers/layer_tree.h

@@ -8,6 +8,7 @@
 #include <cstdint>
 #include <memory>
 
+#include "flutter/common/graphics/texture.h"
 #include "flutter/flow/compositor_context.h"
 #include "flutter/flow/layers/layer.h"
 #include "flutter/flow/raster_cache.h"
@@ -41,7 +42,10 @@ class LayerTree {
   void Paint(CompositorContext::ScopedFrame& frame,
              bool ignore_raster_cache = false) const;
 
-  sk_sp<DisplayList> Flatten(const SkRect& bounds);
+  sk_sp<DisplayList> Flatten(
+      const SkRect& bounds,
+      std::shared_ptr<TextureRegistry> texture_registry = nullptr,
+      GrDirectContext* gr_context = nullptr);
 
   Layer* root_layer() const { return root_layer_.get(); }
 

lib/ui/compositing.dart

@@ -38,6 +38,7 @@ class Scene extends NativeFieldWrapperClass1 {
   external String? _toImageSync(int width, int height, _Image outImage);
 
   /// Creates a raster image representation of the current state of the scene.
+  ///
   /// This is a slow operation that is performed on a background thread.
   ///
   /// Callers must dispose the [Image] when they are done with it. If the result

lib/ui/compositing/scene.cc

@@ -5,6 +5,7 @@
 #include "flutter/lib/ui/compositing/scene.h"
 
 #include "flutter/fml/trace_event.h"
+#include "flutter/lib/ui/painting/display_list_deferred_image_gpu.h"
 #include "flutter/lib/ui/painting/image.h"
 #include "flutter/lib/ui/painting/picture.h"
 #include "flutter/lib/ui/ui_dart_state.h"
@@ -42,7 +43,7 @@ Scene::Scene(std::shared_ptr<flutter::Layer> rootLayer,
                               ->get_window(0)
                               ->viewport_metrics();
 
-  layer_tree_ = std::make_unique<LayerTree>(
+  layer_tree_ = std::make_shared<LayerTree>(
       SkISize::Make(viewport_metrics.physical_width,
                     viewport_metrics.physical_height),
       static_cast<float>(viewport_metrics.device_pixel_ratio));
@@ -69,12 +70,7 @@ Dart_Handle Scene::toImageSync(uint32_t width,
     return tonic::ToDart("Scene did not contain a layer tree.");
   }
 
-  auto picture = layer_tree_->Flatten(SkRect::MakeWH(width, height));
-  if (!picture) {
-    return tonic::ToDart("Could not flatten scene into a layer tree.");
-  }
-
-  Picture::RasterizeToImageSync(picture, width, height, raw_image_handle);
+  Scene::RasterizeToImage(width, height, raw_image_handle);
   return Dart_Null();
 }
 
@@ -87,15 +83,32 @@ Dart_Handle Scene::toImage(uint32_t width,
     return tonic::ToDart("Scene did not contain a layer tree.");
   }
 
-  auto picture = layer_tree_->Flatten(SkRect::MakeWH(width, height));
-  if (!picture) {
-    return tonic::ToDart("Could not flatten scene into a layer tree.");
-  }
+  return Picture::RasterizeLayerTreeToImage(std::move(layer_tree_), width,
+                                            height, raw_image_callback);
+}
 
-  return Picture::RasterizeToImage(picture, width, height, raw_image_callback);
+void Scene::RasterizeToImage(uint32_t width,
+                             uint32_t height,
+                             Dart_Handle raw_image_handle) {
+  auto* dart_state = UIDartState::Current();
+  if (!dart_state) {
+    return;
+  }
+  auto unref_queue = dart_state->GetSkiaUnrefQueue();
+  auto snapshot_delegate = dart_state->GetSnapshotDelegate();
+  auto raster_task_runner = dart_state->GetTaskRunners().GetRasterTaskRunner();
+
+  auto image = CanvasImage::Create();
+  const SkImageInfo image_info = SkImageInfo::Make(
+      width, height, kRGBA_8888_SkColorType, kPremul_SkAlphaType);
+  auto dl_image = DlDeferredImageGPU::MakeFromLayerTree(
+      image_info, std::move(layer_tree_), std::move(snapshot_delegate),
+      std::move(raster_task_runner), std::move(unref_queue));
+  image->set_image(dl_image);
+  image->AssociateWithDartWrapper(raw_image_handle);
 }
 
-std::unique_ptr<flutter::LayerTree> Scene::takeLayerTree() {
+std::shared_ptr<flutter::LayerTree> Scene::takeLayerTree() {
   return std::move(layer_tree_);
 }
 

lib/ui/compositing/scene.h

@@ -26,25 +26,34 @@ class Scene : public RefCountedDartWrappable<Scene> {
                      bool checkerboardRasterCacheImages,
                      bool checkerboardOffscreenLayers);
 
-  std::unique_ptr<flutter::LayerTree> takeLayerTree();
+  std::shared_ptr<flutter::LayerTree> takeLayerTree();
 
   Dart_Handle toImageSync(uint32_t width,
                           uint32_t height,
                           Dart_Handle raw_image_handle);
 
   Dart_Handle toImage(uint32_t width,
                       uint32_t height,
-                      Dart_Handle image_callback);
+                      Dart_Handle raw_image_handle);
 
   void dispose();
 
  private:
-  explicit Scene(std::shared_ptr<flutter::Layer> rootLayer,
-                 uint32_t rasterizerTracingThreshold,
-                 bool checkerboardRasterCacheImages,
-                 bool checkerboardOffscreenLayers);
-
-  std::unique_ptr<flutter::LayerTree> layer_tree_;
+  Scene(std::shared_ptr<flutter::Layer> rootLayer,
+        uint32_t rasterizerTracingThreshold,
+        bool checkerboardRasterCacheImages,
+        bool checkerboardOffscreenLayers);
+
+  void RasterizeToImage(uint32_t width,
+                        uint32_t height,
+                        Dart_Handle raw_image_handle);
+
+  // This is a shared_ptr to support flattening the layer tree from the UI
+  // thread onto the raster thread - allowing access to the texture registry
+  // required to render TextureLayers.
+  //
+  // No longer valid after calling `takeLayerTree`.
+  std::shared_ptr<flutter::LayerTree> layer_tree_;
 };
 
 }  // namespace flutter

lib/ui/painting/display_list_deferred_image_gpu.cc

@@ -22,6 +22,19 @@ sk_sp<DlDeferredImageGPU> DlDeferredImageGPU::Make(
       raster_task_runner));
 }
 
+sk_sp<DlDeferredImageGPU> DlDeferredImageGPU::MakeFromLayerTree(
+    const SkImageInfo& image_info,
+    std::shared_ptr<LayerTree> layer_tree,
+    fml::WeakPtr<SnapshotDelegate> snapshot_delegate,
+    fml::RefPtr<fml::TaskRunner> raster_task_runner,
+    fml::RefPtr<SkiaUnrefQueue> unref_queue) {
+  return sk_sp<DlDeferredImageGPU>(new DlDeferredImageGPU(
+      ImageWrapper::MakeFromLayerTree(
+          image_info, std::move(layer_tree), std::move(snapshot_delegate),
+          raster_task_runner, std::move(unref_queue)),
+      raster_task_runner));
+}
+
 DlDeferredImageGPU::DlDeferredImageGPU(
     std::shared_ptr<ImageWrapper> image_wrapper,
     fml::RefPtr<fml::TaskRunner> raster_task_runner)
@@ -92,6 +105,20 @@ DlDeferredImageGPU::ImageWrapper::Make(
   return wrapper;
 }
 
+std::shared_ptr<DlDeferredImageGPU::ImageWrapper>
+DlDeferredImageGPU::ImageWrapper::MakeFromLayerTree(
+    const SkImageInfo& image_info,
+    std::shared_ptr<LayerTree> layer_tree,
+    fml::WeakPtr<SnapshotDelegate> snapshot_delegate,
+    fml::RefPtr<fml::TaskRunner> raster_task_runner,
+    fml::RefPtr<SkiaUnrefQueue> unref_queue) {
+  auto wrapper = std::shared_ptr<ImageWrapper>(
+      new ImageWrapper(image_info, nullptr, std::move(snapshot_delegate),
+                       std::move(raster_task_runner), std::move(unref_queue)));
+  wrapper->SnapshotDisplayList(std::move(layer_tree));
+  return wrapper;
+}
+
 DlDeferredImageGPU::ImageWrapper::ImageWrapper(
     const SkImageInfo& image_info,
     sk_sp<DisplayList> display_list,
@@ -131,9 +158,11 @@ bool DlDeferredImageGPU::ImageWrapper::isTextureBacked() const {
   return texture_.isValid();
 }
 
-void DlDeferredImageGPU::ImageWrapper::SnapshotDisplayList() {
+void DlDeferredImageGPU::ImageWrapper::SnapshotDisplayList(
+    std::shared_ptr<LayerTree> layer_tree) {
   fml::TaskRunner::RunNowOrPostTask(
-      raster_task_runner_, [weak_this = weak_from_this()]() {
+      raster_task_runner_,
+      [weak_this = weak_from_this(), layer_tree = std::move(layer_tree)]() {
         auto wrapper = weak_this.lock();
         if (!wrapper) {
           return;
@@ -142,6 +171,14 @@ void DlDeferredImageGPU::ImageWrapper::SnapshotDisplayList() {
         if (!snapshot_delegate) {
           return;
         }
+        if (layer_tree) {
+          auto display_list =
+              layer_tree->Flatten(SkRect::MakeWH(wrapper->image_info_.width(),
+                                                 wrapper->image_info_.height()),
+                                  snapshot_delegate->GetTextureRegistry(),
+                                  snapshot_delegate->GetGrContext());
+          wrapper->display_list_ = std::move(display_list);
+        }
         auto result = snapshot_delegate->MakeGpuImage(wrapper->display_list_,
                                                       wrapper->image_info_);
         if (result->texture.isValid()) {

lib/ui/painting/display_list_deferred_image_gpu.h

@@ -11,6 +11,7 @@
 #include "flutter/common/graphics/texture.h"
 #include "flutter/display_list/display_list.h"
 #include "flutter/display_list/display_list_image.h"
+#include "flutter/flow/layers/layer_tree.h"
 #include "flutter/flow/skia_gpu_object.h"
 #include "flutter/fml/macros.h"
 #include "flutter/fml/memory/weak_ptr.h"
@@ -28,6 +29,13 @@ class DlDeferredImageGPU final : public DlImage {
       fml::RefPtr<fml::TaskRunner> raster_task_runner,
       fml::RefPtr<SkiaUnrefQueue> unref_queue);
 
+  static sk_sp<DlDeferredImageGPU> MakeFromLayerTree(
+      const SkImageInfo& image_info,
+      std::shared_ptr<LayerTree> layer_tree,
+      fml::WeakPtr<SnapshotDelegate> snapshot_delegate,
+      fml::RefPtr<fml::TaskRunner> raster_task_runner,
+      fml::RefPtr<SkiaUnrefQueue> unref_queue);
+
   // |DlImage|
   ~DlDeferredImageGPU() override;
 
@@ -73,6 +81,13 @@ class DlDeferredImageGPU final : public DlImage {
         fml::RefPtr<fml::TaskRunner> raster_task_runner,
         fml::RefPtr<SkiaUnrefQueue> unref_queue);
 
+    static std::shared_ptr<ImageWrapper> MakeFromLayerTree(
+        const SkImageInfo& image_info,
+        std::shared_ptr<LayerTree> layer_tree,
+        fml::WeakPtr<SnapshotDelegate> snapshot_delegate,
+        fml::RefPtr<fml::TaskRunner> raster_task_runner,
+        fml::RefPtr<SkiaUnrefQueue> unref_queue);
+
     const SkImageInfo image_info() const { return image_info_; }
     const GrBackendTexture& texture() const { return texture_; }
     bool isTextureBacked() const;
@@ -103,7 +118,11 @@ class DlDeferredImageGPU final : public DlImage {
                  fml::RefPtr<fml::TaskRunner> raster_task_runner,
                  fml::RefPtr<SkiaUnrefQueue> unref_queue);
 
-    void SnapshotDisplayList();
+    // If a layer tree is provided, it will be flattened during the raster
+    // thread task spwaned by this method. After being flattened into a display
+    // list, the image wrapper will be updated to hold this display list and the
+    // layer tree can be dropped.
+    void SnapshotDisplayList(std::shared_ptr<LayerTree> layer_tree = nullptr);
 
     // |ContextListener|
     void OnGrContextCreated() override;

lib/ui/painting/picture.cc

@@ -95,11 +95,21 @@ Dart_Handle Picture::RasterizeToImage(sk_sp<DisplayList> display_list,
                                       Dart_Handle raw_image_callback) {
   return RasterizeToImage(
       [display_list](SkCanvas* canvas) { display_list->RenderTo(canvas); },
-      width, height, raw_image_callback);
+      nullptr, width, height, raw_image_callback);
+}
+
+Dart_Handle Picture::RasterizeLayerTreeToImage(
+    std::shared_ptr<LayerTree> layer_tree,
+    uint32_t width,
+    uint32_t height,
+    Dart_Handle raw_image_callback) {
+  return RasterizeToImage(nullptr, std::move(layer_tree), width, height,
+                          raw_image_callback);
 }
 
 Dart_Handle Picture::RasterizeToImage(
     std::function<void(SkCanvas*)> draw_callback,
+    std::shared_ptr<LayerTree> layer_tree,
     uint32_t width,
     uint32_t height,
     Dart_Handle raw_image_callback) {
@@ -158,10 +168,25 @@ Dart_Handle Picture::RasterizeToImage(
 
   // Kick things off on the raster rask runner.
   fml::TaskRunner::RunNowOrPostTask(
-      raster_task_runner, [ui_task_runner, snapshot_delegate, draw_callback,
-                           picture_bounds, ui_task] {
-        sk_sp<SkImage> raster_image = snapshot_delegate->MakeRasterSnapshot(
-            draw_callback, picture_bounds);
+      raster_task_runner,
+      [ui_task_runner, snapshot_delegate, draw_callback, picture_bounds,
+       ui_task, layer_tree = std::move(layer_tree)] {
+        sk_sp<SkImage> raster_image;
+        if (layer_tree) {
+          auto display_list = layer_tree->Flatten(
+              SkRect::MakeWH(picture_bounds.width(), picture_bounds.height()),
+              snapshot_delegate->GetTextureRegistry(),
+              snapshot_delegate->GetGrContext());
+
+          raster_image = snapshot_delegate->MakeRasterSnapshot(
+              [display_list](SkCanvas* canvas) {
+                display_list->RenderTo(canvas);
+              },
+              picture_bounds);
+        } else {
+          raster_image = snapshot_delegate->MakeRasterSnapshot(draw_callback,
+                                                               picture_bounds);
+        }
 
         fml::TaskRunner::RunNowOrPostTask(
             ui_task_runner,

lib/ui/painting/picture.h

@@ -6,6 +6,7 @@
 #define FLUTTER_LIB_UI_PAINTING_PICTURE_H_
 
 #include "flutter/display_list/display_list.h"
+#include "flutter/flow/layers/layer_tree.h"
 #include "flutter/flow/skia_gpu_object.h"
 #include "flutter/lib/ui/dart_wrapper.h"
 #include "flutter/lib/ui/painting/image.h"
@@ -51,8 +52,19 @@ class Picture : public RefCountedDartWrappable<Picture> {
                                       uint32_t height,
                                       Dart_Handle raw_image_callback);
 
+  static Dart_Handle RasterizeLayerTreeToImage(
+      std::shared_ptr<LayerTree> layer_tree,
+      uint32_t width,
+      uint32_t height,
+      Dart_Handle raw_image_callback);
+
+  // Callers may provide either a draw_callback (which should reference a
+  // display list) or a layer tree. If a layer tree is provided, it will be
+  // flattened on the raster thread. In this case the draw callback will be
+  // ignored.
   static Dart_Handle RasterizeToImage(
       std::function<void(SkCanvas*)> draw_callback,
+      std::shared_ptr<LayerTree> layer_tree,
       uint32_t width,
       uint32_t height,
       Dart_Handle raw_image_callback);