winex11.drv: Support child window vulkan rendering.

Based on a patch from Felix Hädicke <felixhaedicke@web.de>.
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=45277

dlls/winex11.drv/vulkan.c

@@ -32,6 +32,7 @@
 #include "wine/debug.h"
 #include "wine/heap.h"
 #include "x11drv.h"
+#include "xcomposite.h"
 
 #define VK_NO_PROTOTYPES
 #define WINE_VK_HOST
@@ -64,6 +65,7 @@ struct wine_vk_surface
     LONG ref;
     Window window;
     VkSurfaceKHR surface; /* native surface */
+    BOOL offscreen; /* drawable is offscreen */
     HDC dc;
 };
 
@@ -244,6 +246,31 @@ void wine_vk_surface_destroy(HWND hwnd)
     heap_free(surface_list);
 }
 
+static BOOL wine_vk_surface_set_offscreen(struct wine_vk_surface *surface, BOOL offscreen)
+{
+#ifdef SONAME_LIBXCOMPOSITE
+    if (usexcomposite)
+    {
+        if (!surface->offscreen && offscreen)
+        {
+            FIXME("Redirecting vulkan surface offscreen, expect degraded performance.\n");
+            pXCompositeRedirectWindow(gdi_display, surface->window, CompositeRedirectManual);
+        }
+        else if (surface->offscreen && !offscreen)
+        {
+            FIXME("Putting vulkan surface back onscreen, expect standard performance.\n");
+            pXCompositeUnredirectWindow(gdi_display, surface->window, CompositeRedirectManual);
+        }
+        surface->offscreen = offscreen;
+        return TRUE;
+    }
+#endif
+
+    if (offscreen) FIXME("Application requires child window rendering, which is not implemented yet!\n");
+    surface->offscreen = offscreen;
+    return !offscreen;
+}
+
 void resize_vk_surfaces(HWND hwnd, Window active, int mask, XWindowChanges *changes)
 {
     struct wine_vk_surface *surface;
@@ -257,6 +284,17 @@ void resize_vk_surfaces(HWND hwnd, Window active, int mask, XWindowChanges *chan
     LeaveCriticalSection(&context_section);
 }
 
+void sync_vk_surface(HWND hwnd, BOOL known_child)
+{
+    struct wine_vk_surface *surface;
+    struct list *surface_list;
+    EnterCriticalSection(&context_section);
+    if (!XFindContext(gdi_display, (XID)hwnd, vulkan_hwnd_context, (char **)&surface_list))
+        LIST_FOR_EACH_ENTRY(surface, surface_list, struct wine_vk_surface, entry)
+            wine_vk_surface_set_offscreen(surface, known_child);
+    LeaveCriticalSection(&context_section);
+}
+
 static VkResult X11DRV_create_vk_instance_with_callback(const VkInstanceCreateInfo *create_info,
         const VkAllocationCallbacks *allocator, VkInstance *instance,
         VkResult (WINAPI *native_vkCreateInstance)(const VkInstanceCreateInfo *, const VkAllocationCallbacks *,
@@ -295,7 +333,24 @@ static VkResult X11DRV_vkAcquireNextImageKHR(VkDevice device,
         VkSwapchainKHR swapchain, uint64_t timeout, VkSemaphore semaphore,
         VkFence fence, uint32_t *image_index)
 {
-    return pvkAcquireNextImageKHR(device, swapchain, timeout, semaphore, fence, image_index);
+    struct x11drv_escape_present_drawable escape;
+    struct wine_vk_surface *surface;
+    VkResult result;
+
+    EnterCriticalSection(&context_section);
+    if (XFindContext(gdi_display, (XID)swapchain, vulkan_swapchain_context, (char **)&surface)) surface = NULL;
+    LeaveCriticalSection(&context_section);
+
+    result = pvkAcquireNextImageKHR(device, swapchain, timeout, semaphore, fence, image_index);
+    if (result == VK_SUCCESS && surface && surface->offscreen)
+    {
+        escape.code = X11DRV_PRESENT_DRAWABLE;
+        escape.drawable = surface->window;
+        escape.flush = TRUE;
+        ExtEscape(surface->dc, X11DRV_ESCAPE, sizeof(escape), (char *)&escape, 0, NULL);
+    }
+
+    return result;
 }
 
 static VkResult X11DRV_vkAcquireNextImage2KHR(VkDevice device,
@@ -352,13 +407,6 @@ static VkResult X11DRV_vkCreateWin32SurfaceKHR(VkInstance instance,
     if (allocator)
         FIXME("Support for allocation callbacks not implemented yet\n");
 
-    /* TODO: support child window rendering. */
-    if (GetAncestor(create_info->hwnd, GA_PARENT) != GetDesktopWindow())
-    {
-        FIXME("Application requires child window rendering, which is not implemented yet!\n");
-        return VK_ERROR_INCOMPATIBLE_DRIVER;
-    }
-
     x11_surface = heap_alloc_zero(sizeof(*x11_surface));
     if (!x11_surface)
         return VK_ERROR_OUT_OF_HOST_MEMORY;
@@ -377,6 +425,15 @@ static VkResult X11DRV_vkCreateWin32SurfaceKHR(VkInstance instance,
         goto err;
     }
 
+    if (GetWindow( create_info->hwnd, GW_CHILD ) || GetAncestor( create_info->hwnd, GA_PARENT ) != GetDesktopWindow())
+    {
+        if (!wine_vk_surface_set_offscreen(x11_surface, TRUE))
+        {
+            res = VK_ERROR_INCOMPATIBLE_DRIVER;
+            goto err;
+        }
+    }
+
     create_info_host.sType = VK_STRUCTURE_TYPE_XLIB_SURFACE_CREATE_INFO_KHR;
     create_info_host.pNext = NULL;
     create_info_host.flags = 0; /* reserved */
@@ -832,4 +889,8 @@ void resize_vk_surfaces(HWND hwnd, Window active, int mask, XWindowChanges chang
 {
 }
 
+void sync_vk_surface(HWND hwnd, BOOL known_child)
+{
+}
+
 #endif /* SONAME_LIBVULKAN */

dlls/winex11.drv/window.c

@@ -2009,6 +2009,15 @@ void CDECL X11DRV_SetWindowStyle( HWND hwnd, INT offset, STYLESTRUCT *style )
 {
     struct x11drv_win_data *data;
     DWORD changed = style->styleNew ^ style->styleOld;
+    HWND parent = GetAncestor( hwnd, GA_PARENT );
+
+    if (offset == GWL_STYLE && (changed & WS_CHILD))
+    {
+        if (GetWindow( parent, GW_CHILD ) || GetAncestor( parent, GA_PARENT ) != GetDesktopWindow())
+            sync_vk_surface( parent, TRUE );
+        else
+            sync_vk_surface( parent, FALSE );
+    }
 
     if (hwnd == GetDesktopWindow()) return;
     if (!(data = get_win_data( hwnd ))) return;
@@ -2035,6 +2044,10 @@ void CDECL X11DRV_DestroyWindow( HWND hwnd )
 {
     struct x11drv_thread_data *thread_data = x11drv_thread_data();
     struct x11drv_win_data *data;
+    HWND parent = GetAncestor( hwnd, GA_PARENT );
+
+    if (!GetWindow( parent, GW_CHILD ) && GetAncestor( parent, GA_PARENT ) == GetDesktopWindow())
+        sync_vk_surface( parent, FALSE );
 
     if (!(data = get_win_data( hwnd ))) return;
 
@@ -2242,6 +2255,7 @@ static struct x11drv_win_data *X11DRV_create_win_data( HWND hwnd, const RECT *wi
      * that will need clipping support.
      */
     sync_gl_drawable( parent, TRUE );
+    sync_vk_surface( parent, TRUE );
 
     display = thread_init_display();
     init_clip_window();  /* make sure the clip window is initialized in this thread */
@@ -2580,6 +2594,7 @@ void CDECL X11DRV_SetParent( HWND hwnd, HWND parent, HWND old_parent )
      * that will need clipping support.
      */
     sync_gl_drawable( parent, TRUE );
+    sync_vk_surface( parent, TRUE );
 
     fetch_icon_data( hwnd, 0, 0 );
 }

dlls/winex11.drv/x11drv.h

@@ -610,6 +610,7 @@ extern void set_gl_drawable_parent( HWND hwnd, HWND parent ) DECLSPEC_HIDDEN;
 extern void destroy_gl_drawable( HWND hwnd ) DECLSPEC_HIDDEN;
 extern void wine_vk_surface_destroy( HWND hwnd ) DECLSPEC_HIDDEN;
 extern void resize_vk_surfaces( HWND hwnd, Window active, int mask, XWindowChanges *changes ) DECLSPEC_HIDDEN;
+extern void sync_vk_surface( HWND hwnd, BOOL known_child ) DECLSPEC_HIDDEN;
 
 extern void wait_for_withdrawn_state( HWND hwnd, BOOL set ) DECLSPEC_HIDDEN;
 extern Window init_clip_window(void) DECLSPEC_HIDDEN;