Merge pull request #2715 from utmapp/dev

Implement GPU acceleration for OpenGL

.github/workflows/build.yml

@@ -32,6 +32,8 @@ jobs:
             platform: ios
           - arch: x86_64
             platform: ios-tci
+          - arch: arm64
+            platform: ios_simulator # FIXME: enable when ANGLE supports it
     steps:
       - name: Checkout
         uses: actions/checkout@v2

Build.xcconfig

@@ -17,8 +17,8 @@
 // Configuration settings file format documentation can be found at:
 // https://help.apple.com/xcode/#/dev745c5c974
 
-MARKETING_VERSION = 2.1.2
-CURRENT_PROJECT_VERSION = 31
+MARKETING_VERSION = 2.2.0
+CURRENT_PROJECT_VERSION = 32
 
 // Codesigning settings defined optionally, see Documentation/iOSDevelopment.md
 #include? "CodeSigning.xcconfig"

CocoaSpice/CSConnection.h

@@ -31,12 +31,14 @@ NS_ASSUME_NONNULL_BEGIN
 @property (nonatomic, readonly) CSUSBManager *usbManager;
 #endif
 @property (nonatomic, weak, nullable) id<CSConnectionDelegate> delegate;
-@property (nonatomic, copy) NSString *host;
-@property (nonatomic, copy) NSString *port;
+@property (nonatomic, nullable, copy) NSString *host;
+@property (nonatomic, nullable, copy) NSString *port;
+@property (nonatomic, nullable, copy) NSURL *unixSocketURL;
 @property (nonatomic, assign) BOOL audioEnabled;
 
 - (instancetype)init NS_UNAVAILABLE;
 - (instancetype)initWithHost:(NSString *)host port:(NSString *)port NS_DESIGNATED_INITIALIZER;
+- (instancetype)initWithUnixSocketFile:(NSURL *)socketFile NS_DESIGNATED_INITIALIZER;
 - (BOOL)connect;
 - (void)disconnect;
 

CocoaSpice/CSConnection.m

@@ -252,6 +252,11 @@ - (NSString *)port {
     return nshost;
 }
 
+- (void)setUnixSocketURL:(NSURL *)unixSocketURL {
+    g_object_set(self.spiceSession, "unix-path", unixSocketURL.path.UTF8String, NULL);
+    _unixSocketURL = unixSocketURL;
+}
+
 - (void)dealloc {
     UTMLog(@"%s:%d", __FUNCTION__, __LINE__);
     g_signal_handlers_disconnect_by_func(self.spiceSession, G_CALLBACK(cs_channel_new), GLIB_OBJC_RELEASE(self));
@@ -261,27 +266,40 @@ - (void)dealloc {
     self.spiceSession = NULL;
 }
 
+- (void)finishInit {
+    UTMLog(@"%s:%d", __FUNCTION__, __LINE__);
+    g_signal_connect(self.spiceSession, "channel-new",
+                     G_CALLBACK(cs_channel_new), GLIB_OBJC_RETAIN(self));
+    g_signal_connect(self.spiceSession, "channel-destroy",
+                     G_CALLBACK(cs_channel_destroy), GLIB_OBJC_RETAIN(self));
+    g_signal_connect(self.spiceSession, "disconnected",
+                     G_CALLBACK(cs_connection_destroy), GLIB_OBJC_RETAIN(self));
+
+#if !defined(WITH_QEMU_TCI)
+    SpiceUsbDeviceManager *manager = spice_usb_device_manager_get(self.spiceSession, NULL);
+    g_assert(manager != NULL);
+    self.usbManager = [[CSUSBManager alloc] initWithUsbDeviceManager:manager];
+#endif
+    self.input = [[CSInput alloc] initWithSession:self.spiceSession];
+    self.session = [[CSSession alloc] initWithSession:self.spiceSession];
+    self.monitors = [NSArray<CSDisplayMetal *> array];
+}
+
 - (instancetype)initWithHost:(NSString *)host port:(NSString *)port {
     if (self = [super init]) {
         self.spiceSession = spice_session_new();
         self.host = host;
         self.port = port;
-        UTMLog(@"%s:%d", __FUNCTION__, __LINE__);
-        g_signal_connect(self.spiceSession, "channel-new",
-                         G_CALLBACK(cs_channel_new), GLIB_OBJC_RETAIN(self));
-        g_signal_connect(self.spiceSession, "channel-destroy",
-                         G_CALLBACK(cs_channel_destroy), GLIB_OBJC_RETAIN(self));
-        g_signal_connect(self.spiceSession, "disconnected",
-                         G_CALLBACK(cs_connection_destroy), GLIB_OBJC_RETAIN(self));
-
-#if !defined(WITH_QEMU_TCI)
-        SpiceUsbDeviceManager *manager = spice_usb_device_manager_get(self.spiceSession, NULL);
-        g_assert(manager != NULL);
-        self.usbManager = [[CSUSBManager alloc] initWithUsbDeviceManager:manager];
-#endif
-        self.input = [[CSInput alloc] initWithSession:self.spiceSession];
-        self.session = [[CSSession alloc] initWithSession:self.spiceSession];
-        self.monitors = [NSArray<CSDisplayMetal *> array];
+        [self finishInit];
+    }
+    return self;
+}
+
+- (instancetype)initWithUnixSocketFile:(NSURL *)socketFile {
+    if (self = [super init]) {
+        self.spiceSession = spice_session_new();
+        self.unixSocketURL = socketFile;
+        [self finishInit];
     }
     return self;
 }

CocoaSpice/CSDisplayMetal.m

@@ -22,6 +22,7 @@
 #import <glib.h>
 #import <spice-client.h>
 #import <spice/protocol.h>
+#import <IOSurface/IOSurfaceRef.h>
 
 #ifdef DISPLAY_DEBUG
 #undef DISPLAY_DEBUG
@@ -43,10 +44,14 @@ @interface CSDisplayMetal ()
 @property (nonatomic, readwrite) CGPoint cursorHotspot;
 @property (nonatomic, readwrite) BOOL cursorHidden;
 @property (nonatomic, readwrite) BOOL hasCursor;
+@property (nonatomic, readwrite) BOOL isGLEnabled;
+@property (nonatomic, readwrite) BOOL hasGLDrawAck;
+@property (nonatomic) SpiceGlScanout scanout;
 
 // UTMRenderSource properties
 @property (nonatomic) dispatch_queue_t renderQueue;
-@property (nonatomic, nullable, readwrite) id<MTLTexture> displayTexture;
+@property (nonatomic, nullable, readwrite) id<MTLTexture> canvasTexture;
+@property (nonatomic, nullable, readwrite) id<MTLTexture> glTexture;
 @property (nonatomic, nullable, readwrite) id<MTLTexture> cursorTexture;
 @property (nonatomic, readwrite) NSUInteger displayNumVertices;
 @property (nonatomic, readwrite) NSUInteger cursorNumVertices;
@@ -96,6 +101,7 @@ static void cs_invalidate(SpiceChannel *channel,
                        gint x, gint y, gint w, gint h, gpointer data) {
     CSDisplayMetal *self = (__bridge CSDisplayMetal *)data;
     CGRect rect = CGRectIntersection(CGRectMake(x, y, w, h), self->_visibleArea);
+    self.isGLEnabled = NO;
     if (!CGRectIsEmpty(rect)) {
         [self drawRegion:rect];
     }
@@ -151,7 +157,7 @@ static void cs_update_monitor_area(SpiceChannel *channel, GParamSpec *pspec, gpo
     }
 
     /* If only one head on this monitor, update the whole area */
-    if (monitors->len == 1) {
+    if (monitors->len == 1 && !self.isGLEnabled) {
         [self updateVisibleAreaWithRect:CGRectMake(0, 0, c->width, c->height)];
     } else {
         [self updateVisibleAreaWithRect:CGRectMake(c->x, c->y, c->width, c->height)];
@@ -249,6 +255,39 @@ static void cs_cursor_reset(SpiceCursorChannel *channel, gpointer data)
     cs_cursor_invalidate(self);
 }
 
+#pragma mark - GL
+
+static void cs_gl_scanout(SpiceDisplayChannel *channel, GParamSpec *pspec, gpointer data)
+{
+    CSDisplayMetal *self = (__bridge CSDisplayMetal *)data;
+
+    DISPLAY_DEBUG(self, "%s: got scanout",  __FUNCTION__);
+
+    const SpiceGlScanout *scanout;
+
+    scanout = spice_display_channel_get_gl_scanout(self.display);
+    /* should only be called when the display has a scanout */
+    g_return_if_fail(scanout != NULL);
+
+    self.isGLEnabled = YES;
+    self.hasGLDrawAck = YES;
+    self.scanout = *scanout;
+
+    [self rebuildScanoutTexture];
+}
+
+static void cs_gl_draw(SpiceDisplayChannel *channel,
+                       guint32 x, guint32 y, guint32 w, guint32 h,
+                       gpointer data)
+{
+    CSDisplayMetal *self = (__bridge CSDisplayMetal *)data;
+
+    DISPLAY_DEBUG(self, "%s",  __FUNCTION__);
+
+    self.isGLEnabled = YES;
+    self.hasGLDrawAck = NO;
+}
+
 #pragma mark - Channel events
 
 static void cs_channel_new(SpiceSession *s, SpiceChannel *channel, gpointer data) {
@@ -276,6 +315,10 @@ static void cs_channel_new(SpiceSession *s, SpiceChannel *channel, gpointer data
                                G_CALLBACK(cs_update_monitor_area), GLIB_OBJC_RETAIN(self));
         g_signal_connect_after(channel, "gst-video-overlay",
                                G_CALLBACK(cs_set_overlay), GLIB_OBJC_RETAIN(self));
+        g_signal_connect(channel, "notify::gl-scanout",
+                         G_CALLBACK(cs_gl_scanout), GLIB_OBJC_RETAIN(self));
+        g_signal_connect(channel, "gl-draw",
+                         G_CALLBACK(cs_gl_draw), GLIB_OBJC_RETAIN(self));
         if (spice_display_channel_get_primary(channel, 0, &primary)) {
             cs_primary_create(channel, primary.format, primary.width, primary.height,
                               primary.stride, primary.shmid, primary.data, (__bridge void *)self);
@@ -339,6 +382,8 @@ static void cs_channel_destroy(SpiceSession *s, SpiceChannel *channel, gpointer
         g_signal_handlers_disconnect_by_func(channel, G_CALLBACK(cs_mark), GLIB_OBJC_RELEASE(self));
         g_signal_handlers_disconnect_by_func(channel, G_CALLBACK(cs_update_monitor_area), GLIB_OBJC_RELEASE(self));
         g_signal_handlers_disconnect_by_func(channel, G_CALLBACK(cs_set_overlay), GLIB_OBJC_RELEASE(self));
+        g_signal_handlers_disconnect_by_func(channel, G_CALLBACK(cs_gl_scanout), GLIB_OBJC_RELEASE(self));
+        g_signal_handlers_disconnect_by_func(channel, G_CALLBACK(cs_gl_draw), GLIB_OBJC_RELEASE(self));
         return;
     }
 
@@ -365,8 +410,12 @@ static void cs_channel_destroy(SpiceSession *s, SpiceChannel *channel, gpointer
 
 - (void)setDevice:(id<MTLDevice>)device {
     _device = device;
-    [self rebuildDisplayTexture];
     [self rebuildDisplayVertices];
+    if (self.isGLEnabled) {
+        [self rebuildScanoutTexture];
+    } else {
+        [self rebuildCanvasTexture];
+    }
 }
 
 - (UTMScreenshot *)screenshot {
@@ -377,6 +426,9 @@ - (UTMScreenshot *)screenshot {
         CGDataProviderRef dataProviderRef = CGDataProviderCreateWithData(NULL, _canvasData, _canvasStride * _canvasArea.size.height, nil);
         img = CGImageCreate(_canvasArea.size.width, _canvasArea.size.height, 8, 32, _canvasStride, colorSpaceRef, kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipFirst, dataProviderRef, NULL, NO, kCGRenderingIntentDefault);
         CGDataProviderRelease(dataProviderRef);
+    } else if (_glTexture) {
+        // TODO: make screenshot from IOSurface
+        img = NULL;
     } else {
         img = NULL;
     }
@@ -396,6 +448,14 @@ - (UTMScreenshot *)screenshot {
     }
 }
 
+- (id<MTLTexture>)displayTexture {
+    if (self.isGLEnabled) {
+        return self.glTexture;
+    } else {
+        return self.canvasTexture;
+    }
+}
+
 #pragma mark - Methods
 
 - (instancetype)initWithSession:(nonnull SpiceSession *)session channelID:(NSInteger)channelID monitorID:(NSInteger)monitorID {
@@ -452,7 +512,13 @@ - (void)dealloc {
 }
 
 - (void)updateVisibleAreaWithRect:(CGRect)rect {
-    CGRect visible = CGRectIntersection(_canvasArea, rect);
+    CGRect primary;
+    if (self.isGLEnabled) {
+        primary = CGRectMake(0, 0, self.scanout.width, self.scanout.height);
+    } else {
+        primary = _canvasArea;
+    }
+    CGRect visible = CGRectIntersection(primary, rect);
     if (CGRectIsNull(visible)) {
         DISPLAY_DEBUG(self, "The monitor area is not intersecting primary surface");
         self.ready = NO;
@@ -461,11 +527,37 @@ - (void)updateVisibleAreaWithRect:(CGRect)rect {
         _visibleArea = visible;
     }
     self.displaySize = _visibleArea.size;
-    [self rebuildDisplayTexture];
     [self rebuildDisplayVertices];
+    if (!self.isGLEnabled) {
+        [self rebuildCanvasTexture];
+    }
+}
+
+- (void)rebuildScanoutTexture {
+    if (!self.device) {
+        return; // not ready
+    }
+    IOSurfaceID iosurfaceid = 0;
+    IOSurfaceRef iosurface = NULL;
+    if (read(self.scanout.fd, &iosurfaceid, sizeof(iosurfaceid)) != sizeof(iosurfaceid)) {
+        UTMLog(@"Failed to read scanout fd: %d", self.scanout.fd);
+        perror("read");
+        return;
+    }
+    if ((iosurface = IOSurfaceLookup(iosurfaceid)) == NULL) {
+        UTMLog(@"Failed to lookup surface: %d", iosurfaceid);
+        return;
+    }
+
+    MTLTextureDescriptor *textureDescriptor = [[MTLTextureDescriptor alloc] init];
+    textureDescriptor.pixelFormat = MTLPixelFormatBGRA8Unorm;
+    textureDescriptor.width = self.scanout.width;
+    textureDescriptor.height = self.scanout.height;
+    self.glTexture = [self.device newTextureWithDescriptor:textureDescriptor iosurface:iosurface plane:0];
+    CFRelease(iosurface);
 }
 
-- (void)rebuildDisplayTexture {
+- (void)rebuildCanvasTexture {
     if (CGRectIsEmpty(_canvasArea) || !self.device) {
         return;
     }
@@ -474,7 +566,7 @@ - (void)rebuildDisplayTexture {
     textureDescriptor.pixelFormat = (_canvasFormat == SPICE_SURFACE_FMT_32_xRGB) ? MTLPixelFormatBGRA8Unorm : (MTLPixelFormat)43;// FIXME: MTLPixelFormatBGR5A1Unorm is supposed to be available.
     textureDescriptor.width = _visibleArea.size.width;
     textureDescriptor.height = _visibleArea.size.height;
-    self.displayTexture = [self.device newTextureWithDescriptor:textureDescriptor];
+    self.canvasTexture = [self.device newTextureWithDescriptor:textureDescriptor];
     [self drawRegion:_visibleArea];
 }
 
@@ -512,7 +604,7 @@ - (void)drawRegion:(CGRect)rect {
     gint canvasStride = _canvasStride;
     if (canvasData) {
         dispatch_async(self.renderQueue, ^{
-            [self.displayTexture replaceRegion:region
+            [self.canvasTexture  replaceRegion:region
                                    mipmapLevel:0
                                      withBytes:(const char *)canvasData + (NSUInteger)(rect.origin.y*canvasStride + rect.origin.x*pixelSize)
                                    bytesPerRow:canvasStride];
@@ -540,6 +632,13 @@ - (void)requestResolution:(CGRect)bounds {
     spice_main_channel_send_monitor_config(self.main);
 }
 
+- (void)rendererFrameHasRendered {
+    if (self.isGLEnabled && !self.hasGLDrawAck) {
+        spice_display_channel_gl_draw_done(self.display);
+        self.hasGLDrawAck = YES;
+    }
+}
+
 #pragma mark - Cursor drawing
 
 - (void)rebuildCursorWithSize:(CGSize)size center:(CGPoint)hotspot {

CocoaSpice/CSMain.m

@@ -114,7 +114,9 @@ - (void)dealloc {
 
 - (void)spiceSetDebug:(BOOL)enabled {
     spice_util_set_debug(enabled);
-    g_log_set_handler(NULL, G_LOG_LEVEL_MASK, logHandler, NULL);
+    if (enabled) {
+        g_log_set_handler(NULL, G_LOG_LEVEL_MASK, logHandler, NULL);
+    }
 }
 
 - (BOOL)spiceStart {