Merge pull request macvim-dev#1521 from ychin/fullscreen-tests-fuopt-…

…fixes

Contains the following:

1. Fix non-native full screen misc background color and transparency issues.
2. Fix non-native full screen bad interaction with window delegate.
3. Add tests for full screen code

runtime/doc/options.txt

@@ -3981,15 +3981,19 @@ A jump table for the options with a short description can be found at |Q_op|.
 		of columns fitting on the screen in fullscreen mode. If unset,
 		'columns' will be unchanged when entering fullscreen mode.
 	background:color
-		When entering fullscreen, 'color' defines the color of the part
-		of the screen that is not occupied by the Vim control. If
-		'color' is an 8-digit hexadecimal number preceded by '#',
-		it is interpreted as an explicit color value '#aarrggbb', with
-		one byte each for the alpha, red, green, and blue values.
-		Otherwise, 'color' is interpreted as a highlight group name,
+		When entering fullscreen, "color" defines the color of the part
+		of the screen that is not occupied by the Vim control,
+		including the "notch" area for MacBook laptops.
+		If "color" is a 6 or 8-digit hexadecimal number preceded by
+		'#', it is interpreted as an explicit color value '#rrggbb' /
+		'#aarrggbb', with one byte each for the alpha, red, green, and
+		blue values. The alpha value is ignored (use 'transparency'
+		instead for a translucent window).
+		Otherwise, "color" is interpreted as a highlight group name,
 		and the fullscreen background is filled with that highlight
 		group's background color, as defined by the current color
 		scheme.
+		If unset, the default value is black (#FF000000).
 
 	Examples:
 	Don't change size when entering fullscreen: >

src/MacVim/MMBackend.m

@@ -1188,7 +1188,7 @@ - (void)leaveFullScreen
 - (void)setFullScreenBackgroundColor:(int)color
 {
     NSMutableData *data = [NSMutableData data];
-    color = MM_COLOR(color);
+    color = MM_COLOR_WITH_TRANSP(color,p_transp);
     [data appendBytes:&color length:sizeof(int)];
 
     [self queueMessage:SetFullScreenColorMsgID data:data];

src/MacVim/MMCoreTextView.m

@@ -831,7 +831,7 @@ - (void)drawRect:(NSRect)rect
 
     // Function to draw all rows
     void (^drawAllRows)(void (^)(CGContextRef,CGRect,int)) = ^(void (^drawFunc)(CGContextRef,CGRect,int)){
-        for (size_t r = 0; r < grid.rows; r++) {
+        for (int r = 0; r < grid.rows; r++) {
             const CGRect rowRect = [self rectForRow:(int)r
                                              column:0
                                             numRows:1
@@ -1276,6 +1276,9 @@ - (NSSize)constrainRows:(int *)rows columns:(int *)cols toSize:(NSSize)size
         if (fh < 1.0f) fh = 1.0f;
 
         desiredRows = floor((size.height - ih)/fh);
+        // Sanity checking in case unusual window sizes lead to degenerate results
+        if (desiredRows < 1)
+            desiredRows = 1;
         desiredSize.height = fh*desiredRows + ih;
     }
 
@@ -1285,6 +1288,9 @@ - (NSSize)constrainRows:(int *)rows columns:(int *)cols toSize:(NSSize)size
         if (fw < 1.0f) fw = 1.0f;
 
         desiredCols = floor((size.width - iw)/fw);
+        // Sanity checking in case unusual window sizes lead to degenerate results
+        if (desiredCols < 1)
+            desiredCols = 1;
         desiredSize.width = fw*desiredCols + iw;
     }
 

src/MacVim/MMFullScreenWindow.m

@@ -136,6 +136,15 @@ - (void)enterFullScreen
 {
     ASLogDebug(@"Enter full-screen now");
 
+    // Detach the window delegate right now to prevent any stray window
+    // messages (e.g. it may get resized when setting presentationOptions
+    // below) being sent to the window controller while we are in the middle of
+    // setting up the full screen window.
+    NSWindowController *winController = [target windowController];
+    id delegate = [target delegate];
+    [winController setWindow:nil];
+    [target setDelegate:nil];
+
     // Hide Dock and menu bar when going to full screen. Only do so if the current screen
     // has a menu bar and dock.
     if ([self screenHasDockAndMenu]) {
@@ -162,13 +171,6 @@ - (void)enterFullScreen
     // this call so set the frame again just in case.
     [self setFrame:[[target screen] frame] display:NO];
 
-    // fool delegate
-    id delegate = [target delegate];
-    [target setDelegate:nil];
-
-    // make target's window controller believe that it's now controlling us
-    [[target windowController] setWindow:self];
-
     oldTabBarStyle = [[view tabBarControl] styleName];
 
     NSString *style =
@@ -181,7 +183,7 @@ - (void)enterFullScreen
     [view removeFromSuperviewWithoutNeedingDisplay];
     [[self contentView] addSubview:view];
     [self setInitialFirstResponder:[view textView]];
-    
+
     // NOTE: Calling setTitle:nil causes an exception to be raised (and it is
     // possible that 'target' has no title when we get here).
     if ([target title]) {
@@ -196,8 +198,10 @@ - (void)enterFullScreen
 
     [self setOpaque:[target isOpaque]];
 
+    // reassign target's window controller to believe that it's now controlling us
     // don't set this sooner, so we don't get an additional
-    // focus gained message  
+    // focus gained message
+    [winController setWindow:self];
     [self setDelegate:delegate];
 
     // Store view dimension used before entering full-screen, then resize the

src/MacVim/MMTextStorage.m

@@ -895,6 +895,9 @@ - (NSSize)fitToSize:(NSSize)size rows:(int *)rows columns:(int *)columns
         if (fh < 1.0f) fh = 1.0f;
 
         fitRows = floor(size.height/fh);
+        // Sanity checking in case unusual window sizes lead to degenerate results
+        if (fitRows < 1)
+            fitRows = 1;
         fitSize.height = fh*fitRows;
     }
 
@@ -903,6 +906,9 @@ - (NSSize)fitToSize:(NSSize)size rows:(int *)rows columns:(int *)columns
         if (fw < 1.0f) fw = 1.0f;
 
         fitCols = floor(size.width/fw);
+        // Sanity checking in case unusual window sizes lead to degenerate results
+        if (fitCols < 1)
+            fitCols = 1;
         fitSize.width = fw*fitCols;
     }
 

src/MacVim/MMVimController.m

@@ -1129,7 +1129,7 @@ - (void)handleMessage:(int)msgid data:(NSData *)data
         case SetFullScreenColorMsgID:
         {
             const int *bg = (const int*)[data bytes];
-            NSColor *color = [NSColor colorWithRgbInt:*bg];
+            NSColor *color = [NSColor colorWithArgbInt:*bg];
 
             [windowController setFullScreenBackgroundColor:color];
         }

src/MacVim/MMWindowController.m

@@ -699,25 +699,32 @@ - (void)setDefaultColorsBackground:(NSColor *)back foreground:(NSColor *)fore
         // window, so we need to set a transparency color here to make the
         // transparency show through.
         if ([back alphaComponent] == 1) {
-            // Here, any solid color would do, but setting it with "back" has an
-            // interesting effect where the title bar gets subtly tinted by it
-            // as well, so do that. (Note that this won't play well in <=10.12
-            // since we are using the deprecated
-            // NSWindowStyleMaskTexturedBackground which makes the titlebars
-            // transparent in those. Consider not using textured background.)
+            // The window's background color affects the title bar tint and
+            // if we are using a transparent title bar this color will show
+            // up as well.
+            // (Note that this won't play well in <=10.12 since we are using
+            // the deprecated NSWindowStyleMaskTexturedBackground which makes
+            // the titlebars transparent in those. Consider not using textured
+            // background.)
             [decoratedWindow setBackgroundColor:back];
+
+            // Note: We leave the full screen window's background color alone
+            // because it is affected by 'fuoptions' instead. We just change the
+            // alpha back to 1 in case it was changed previously because transparency
+            // was set.
             if (fullScreenWindow) {
-                [fullScreenWindow setBackgroundColor:back];
+                [fullScreenWindow setBackgroundColor:
+                 [fullScreenWindow.backgroundColor colorWithAlphaComponent:1]];
             }
         } else {
             // HACK! We really want a transparent background color to avoid
             // double blending the transparency, but setting alpha=0 leads to
             // the window border disappearing and also drag-to-resize becomes a
             // lot slower. So hack around it by making it virtually transparent.
-            NSColor *clearColor = [back colorWithAlphaComponent:0.001];
-            [decoratedWindow setBackgroundColor:clearColor];
+            [decoratedWindow setBackgroundColor:[back colorWithAlphaComponent:0.001]];
             if (fullScreenWindow) {
-                [fullScreenWindow setBackgroundColor:clearColor];
+                [fullScreenWindow setBackgroundColor:
+                 [fullScreenWindow.backgroundColor colorWithAlphaComponent:0.001]];
             }
         }
     }
@@ -1046,9 +1053,17 @@ - (void)leaveFullScreen
     }
 }
 
+/// Called when the window is in non-native full-screen mode and the user has
+/// updated the background color.
 - (void)setFullScreenBackgroundColor:(NSColor *)back
 {
     if (fullScreenWindow)
+        // See setDefaultColorsBackground: for why set a transparent
+        // background color, and why 0.001 instead of 0.
+        if ([back alphaComponent] != 1) {
+            back = [back colorWithAlphaComponent:0.001];
+        }
+
         [fullScreenWindow setBackgroundColor:back];
 }
 
@@ -1307,9 +1322,7 @@ - (void)windowDidResize:(id)sender
     // Calling setFrameSizeKeepGUISize: instead of setFrameSize: prevents a
     // degenerate case where frameSizeMayHaveChanged: ends up resizing the window
     // *again* causing windowDidResize: to be called.
-    if (fullScreenWindow == nil) {
-        [vimView setFrameSizeKeepGUISize:[self contentSize]];
-    } else {
+    if (fullScreenEnabled && fullScreenWindow != nil) {
         // Non-native full screen mode is more complicated and needs to
         // re-layout the Vim view to properly account for the menu bar / notch,
         // and misc fuopt configuration.
@@ -1318,6 +1331,9 @@ - (void)windowDidResize:(id)sender
         [vimView setFrameOrigin:desiredFrame.origin];
         [vimView setFrameSizeKeepGUISize:desiredFrame.size];
     }
+    else {
+        [vimView setFrameSizeKeepGUISize:[self contentSize]];
+    }
 }
 
 - (void)windowDidChangeBackingProperties:(NSNotification *)notification