MMTabs: Use TabLineFill fg, fix colorscheme w/ transparent/inverse highlights

When using Vim colorscheme mode, fix the fill foreground colors (used
for add tab and scroll buttons) to use the colorscheme's TabLineFill's
guifg colors. Previously they were automatically calculated based on the
other foreground colors which worked for automatic colors but not when
we are using a colorscheme where it may not match. Just use the one
specified in color scheme. Colorschemes like everforest and koehler
should have visible buttons now.

Also, fix the previous broken logic for parsing the colorscheme colors.
It was not accounting for transparent colors where missing colors or
cleared highlight groups should fall through to default bg/fg colors
(e.g. delek and catppuccin do this for TabLineSel). It also wasn't
respecting the `gui=reverse` flag (which means bg/fg colors should be
flipped) that some colorschemes (e.g. Solarized) uses.

Author: ychin

src/MacVim/MMTabline/MMTabline.h

@@ -20,13 +20,15 @@
 @property (nonatomic, readonly) NSInteger numberOfTabs;
 @property (nonatomic, retain, readonly) MMHoverButton *addTabButton;
 
-@property (nonatomic, retain) NSColor *tablineBgColor;
-@property (nonatomic, retain) NSColor *tablineFgColor;
-@property (nonatomic, retain) NSColor *tablineSelBgColor;
-@property (nonatomic, retain) NSColor *tablineSelFgColor;
-@property (nonatomic, retain) NSColor *tablineFillFgColor;
-
-// Derived colors that cannot be set directly
+// Main colors
+@property (nonatomic, readonly) NSColor *tablineBgColor;
+@property (nonatomic, readonly) NSColor *tablineFgColor;
+@property (nonatomic, readonly) NSColor *tablineSelBgColor;
+@property (nonatomic, readonly) NSColor *tablineSelFgColor;
+@property (nonatomic, readonly) NSColor *tablineFillBgColor;
+@property (nonatomic, readonly) NSColor *tablineFillFgColor;
+
+// Derived colors from the main ones
 @property (nonatomic, readonly) NSColor *tablineUnfocusedFgColor;
 @property (nonatomic, readonly) NSColor *tablineUnfocusedSelFgColor;
 @property (nonatomic, readonly) NSColor *tablineStrokeColor;
@@ -69,7 +71,7 @@
 /// colors based on the system light/dark modes.
 - (void)setColorsTabBg:(NSColor *)tabBg tabFg:(NSColor *)tabFg
                  selBg:(NSColor *)selBg selFg:(NSColor *)selFg
-                  fill:(NSColor *)fill;
+                fillBg:(NSColor *)fill fillFg:(NSColor *)fillFg;
 
 /// Lets the tabline calculate best colors to use based on background and
 /// foreground colors of the selected tab. The colors cannot be nil.

src/MacVim/MMTabline/MMTabline.m

@@ -67,7 +67,7 @@ @implementation MMTabline
 @synthesize tablineFgColor = _tablineFgColor;
 @synthesize tablineSelBgColor = _tablineSelBgColor;
 @synthesize tablineSelFgColor = _tablineSelFgColor;
-@synthesize tablineFillFgColor = _tablineFillFgColor;
+@synthesize tablineFillBgColor = _tablineFillBgColor;
 @synthesize tablineUnfocusedFgColor = _tablineUnfocusedFgColor;
 @synthesize tablineUnfocusedSelFgColor = _tablineUnfocusedSelFgColor;
 
@@ -150,7 +150,7 @@ - (BOOL)wantsUpdateLayer { return YES; }
 
 - (void)updateLayer
 {
-    self.layer.backgroundColor = self.tablineFillFgColor.CGColor;
+    self.layer.backgroundColor = self.tablineFillBgColor.CGColor;
 }
 
 - (void)viewDidChangeEffectiveAppearance
@@ -253,63 +253,33 @@ - (NSColor *)tablineBgColor
     }
 }
 
-- (void)setTablineBgColor:(NSColor *)color
-{
-    _tablineBgColor = color;
-    [self updateTabStates];
-}
-
 - (NSColor *)tablineFgColor
 {
     return _tablineFgColor ?: NSColor.secondaryLabelColor;
 }
 
-- (void)setTablineFgColor:(NSColor *)color
-{
-    _tablineFgColor = color;
-    _tablineUnfocusedFgColor = nil;
-    [self updateTabStates];
-}
-
 - (NSColor *)tablineSelBgColor
 {
     return _tablineSelBgColor ?: (_appearance == AppearanceLight || _appearance == AppearanceLightHighContrast)
         ? [NSColor colorWithWhite:0.95 alpha:1]
         : [NSColor colorWithWhite:0.4 alpha:1];
 }
 
-- (void)setTablineSelBgColor:(NSColor *)color
-{
-    _tablineSelBgColor = color;
-    [self updateTabStates];
-}
-
 - (NSColor *)tablineSelFgColor
 {
     return _tablineSelFgColor ?: NSColor.controlTextColor;
 }
 
-- (void)setTablineSelFgColor:(NSColor *)color
-{
-    _tablineSelFgColor = color;
-    _tablineUnfocusedSelFgColor = nil;
-    _addTabButton.fgColor = color;
-    _backwardScrollButton.fgColor = color;
-    _forwardScrollButton.fgColor = color;
-    [self updateTabStates];
-}
-
-- (NSColor *)tablineFillFgColor
+- (NSColor *)tablineFillBgColor
 {
-    return _tablineFillFgColor ?: (_appearance == AppearanceLight || _appearance == AppearanceLightHighContrast)
+    return _tablineFillBgColor ?: (_appearance == AppearanceLight || _appearance == AppearanceLightHighContrast)
         ? [NSColor colorWithWhite:0.85 alpha:1]
         : [NSColor colorWithWhite:0.23 alpha:1];
 }
 
-- (void)setTablineFillFgColor:(NSColor *)color
+- (NSColor *)tablineFillFgColor
 {
-    _tablineFillFgColor = color;
-    self.needsDisplay = YES;
+    return _addTabButton.fgColor;
 }
 
 - (NSColor *)tablineUnfocusedFgColor
@@ -570,22 +540,22 @@ - (MMTab *)tabAtIndex:(NSInteger)index
 
 - (void)setColorsTabBg:(NSColor *)tabBg tabFg:(NSColor *)tabFg
                  selBg:(NSColor *)selBg selFg:(NSColor *)selFg
-                  fill:(NSColor *)fill
+                fillBg:(NSColor *)fillBg fillFg:(NSColor *)fillFg
 {
     // Don't use the property mutators as we just want to update the states in
     // one go at the end.
     _tablineSelBgColor = selBg;
     _tablineSelFgColor = selFg;
     _tablineBgColor = tabBg;
     _tablineFgColor = tabFg;
-    _tablineFillFgColor = fill;
+    _tablineFillBgColor = fillBg;
 
     _tablineUnfocusedFgColor = [_tablineFgColor blendedColorWithFraction:0.4 ofColor:_tablineBgColor];
     _tablineUnfocusedSelFgColor = [_tablineSelFgColor blendedColorWithFraction:0.38 ofColor:_tablineSelBgColor];
 
-    _addTabButton.fgColor = _tablineSelFgColor;
-    _backwardScrollButton.fgColor = _tablineSelFgColor;
-    _forwardScrollButton.fgColor = _tablineSelFgColor;
+    _addTabButton.fgColor = fillFg;
+    _backwardScrollButton.fgColor = fillFg;
+    _forwardScrollButton.fgColor = fillFg;
 
     [self updateTabStates];
     self.needsDisplay = YES;
@@ -615,7 +585,7 @@ - (void)setAutoColorsSelBg:(NSColor *)back fg:(NSColor *)fore;
     _tablineUnfocusedFgColor = [_tablineFgColor blendedColorWithFraction:0.4 ofColor:_tablineBgColor];
     _tablineUnfocusedSelFgColor = [_tablineSelFgColor blendedColorWithFraction:0.38 ofColor:_tablineSelBgColor];
 
-    _tablineFillFgColor = (brightness > 0.5)
+    _tablineFillBgColor = (brightness > 0.5)
         ? [back blendedColorWithFraction:0.25 ofColor:NSColor.blackColor]
         : [back blendedColorWithFraction:0.18 ofColor:NSColor.whiteColor];
 

src/MacVim/MMVimView.m

@@ -40,7 +40,8 @@
     MMTabColorTypeTabFg,
     MMTabColorTypeSelBg,
     MMTabColorTypeSelFg,
-    MMTabColorTypeFill,
+    MMTabColorTypeFillBg,
+    MMTabColorTypeFillFg,
     MMTabColorTypeCount
 } MMTabColorType;
 
@@ -377,7 +378,7 @@ - (void)refreshTabProperties
 {
     NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
     tabline.showsTabScrollButtons = [ud boolForKey:MMShowTabScrollButtonsKey];
-    [self updateTablineColors];
+    [self updateTablineColors:MMTabColorsModeCount];
 }
 
 - (void)createScrollbarWithIdentifier:(int32_t)ident type:(int)type
@@ -469,25 +470,35 @@ - (void)finishPlaceScrollbars
     }
 }
 
-- (void)updateTablineColors
+- (void)updateTablineColors:(MMTabColorsMode)mode
 {
     NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
     MMTabColorsMode tabColorsMode = [ud integerForKey:MMTabColorsModeKey];
+    if (tabColorsMode >= MMTabColorsModeCount || tabColorsMode < 0) {
+        // Catch-all for invalid values, which could be useful if we add new
+        // modes and a user goes back and uses an old version of MacVim.
+        tabColorsMode = MMTabColorsModeAutomatic;
+    }
+    if (mode != MMTabColorsModeCount && mode != tabColorsMode) {
+        // Early out to avoid unnecessary updates if this is not relevant.
+        return;
+    }
     if (tabColorsMode == MMTabColorsModeDefaultColors) {
         [tabline setColorsTabBg:nil
                           tabFg:nil
                           selBg:nil
                           selFg:nil
-                           fill:nil];
+                         fillBg:nil
+                         fillFg:nil];
     } else if (tabColorsMode == MMTabColorsModeVimColorscheme) {
         [tabline setColorsTabBg:tabColors[MMTabColorTypeTabBg]
                           tabFg:tabColors[MMTabColorTypeTabFg]
                           selBg:tabColors[MMTabColorTypeSelBg]
                           selFg:tabColors[MMTabColorTypeSelFg]
-                           fill:tabColors[MMTabColorTypeFill]];
+                         fillBg:tabColors[MMTabColorTypeFillBg]
+                         fillFg:tabColors[MMTabColorTypeFillFg]];
     } else {
-        // tabColorsMode == MMTabColorsModeAutomatic, but catch-all in case it's
-        // set to an out-of-range number.
+        // tabColorsMode == MMTabColorsModeAutomatic
         NSColor *back = [[self textView] defaultBackgroundColor];
         NSColor *fore = [[self textView] defaultForegroundColor];
         [tabline setAutoColorsSelBg:back fg:fore];
@@ -498,7 +509,7 @@ - (void)updateTablineColors
 - (void)setDefaultColorsBackground:(NSColor *)back foreground:(NSColor *)fore
 {
     [textView setDefaultColorsBackground:back foreground:fore];
-    [self updateTablineColors];
+    [self updateTablineColors:MMTabColorsModeAutomatic];
 
     CALayer *backedLayer = [self layer];
     if (backedLayer) {
@@ -527,9 +538,9 @@ - (void)setTablineColorsTabBg:(NSColor *)tabBg tabFg:(NSColor *)tabFg
     tabColors[MMTabColorTypeTabFg] = [tabFg retain];
     tabColors[MMTabColorTypeSelBg] = [selBg retain];
     tabColors[MMTabColorTypeSelFg] = [selFg retain];
-    tabColors[MMTabColorTypeFill] = [fillBg retain];
-    (void)fillFg; // We don't use fillFg as we don't draw anything in the empty area
-    [self updateTablineColors];
+    tabColors[MMTabColorTypeFillBg] = [fillBg retain];
+    tabColors[MMTabColorTypeFillFg] = [fillFg retain];
+    [self updateTablineColors:MMTabColorsModeVimColorscheme];
 }
 
 

src/MacVim/MMWindowController.m

@@ -743,7 +743,7 @@ - (void)setTablineColorsTabBg:(NSColor *)tabBg tabFg:(NSColor *)tabFg
 - (void)setWindowColorToTablineColor
 {
     NSColor *defaultBg = vimView.textView.defaultBackgroundColor;
-    NSColor *tablineColor = vimView.tabline.tablineFillFgColor;
+    NSColor *tablineColor = vimView.tabline.tablineFillBgColor;
     if (defaultBg.alphaComponent == 1.0) {
         [self setWindowBackgroundColor:tablineColor];
     } else {

src/MacVim/MacVimTests/MacVimTests.m

@@ -33,6 +33,10 @@ @interface MMVimController (Private)
 - (void)handleMessage:(int)msgid data:(NSData *)data;
 @end
 
+@interface MMVimView (Tests)
+- (void)updateTablineColors:(MMTabColorsMode)mode;
+@end
+
 // Test harness
 @implementation MMAppController (Tests)
 - (NSMutableArray*)vimControllers {
@@ -954,6 +958,63 @@ - (void) testResizeVimView {
     XCTAssertFalse(win.isRenderBlocked);
 }
 
+#pragma mark Tabs tests
+
+- (void)testTabColors {
+    [self createTestVimWindow];
+
+    MMAppController *app = MMAppController.sharedInstance;
+    MMVimView *vimView = [[[app keyVimController] windowController] vimView];
+    MMTabline *tabline = [vimView tabline];
+
+    // Test Vim colorscheme mode
+    [self setDefault:MMTabColorsModeKey toValue:@(MMTabColorsModeVimColorscheme)];
+
+    [self sendStringToVim:@":hi Normal guifg=#ff0000 guibg=#00ff00\n" withMods:0];
+    [self waitForVimProcess];
+    [self sendStringToVim:@":hi TabLineSel guifg=#010203 guibg=#040506\n" withMods:0];
+    [self sendStringToVim:@":hi clear TabLineFill\n" withMods:0];
+    [self sendStringToVim:@":hi TabLine guifg=#111213 guibg=NONE gui=inverse\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+
+    // Normal highlight groups
+    XCTAssertEqualObjects(tabline.tablineSelBgColor, [NSColor colorWithRgbInt:0x040506]);
+    XCTAssertEqualObjects(tabline.tablineSelFgColor, [NSColor colorWithRgbInt:0x010203]);
+    // Cleared highlight group should be transparent and fall through to Normal group
+    XCTAssertEqualObjects(tabline.tablineFillBgColor, [NSColor colorWithRgbInt:0x00ff00]);
+    XCTAssertEqualObjects(tabline.tablineFillFgColor, [NSColor colorWithRgbInt:0xff0000]);
+    // One color is transparent, and inversed fg/bg
+    XCTAssertEqualObjects(tabline.tablineBgColor, [NSColor colorWithRgbInt:0x111213]);
+    XCTAssertEqualObjects(tabline.tablineFgColor, [NSColor colorWithRgbInt:0x00ff00]);
+
+    // Cleared highlight group with inversed fg/bg
+    [self sendStringToVim:@":hi TabLineFill gui=inverse\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(tabline.tablineFillBgColor, [NSColor colorWithRgbInt:0xff0000]);
+    XCTAssertEqualObjects(tabline.tablineFillFgColor, [NSColor colorWithRgbInt:0x00ff00]);
+
+    // Test automatic colors mode
+    // Selected tab should have the exact same background as Normal colors
+    [self setDefault:MMTabColorsModeKey toValue:@(MMTabColorsModeAutomatic)];
+    [vimView updateTablineColors:MMTabColorsModeAutomatic];
+    XCTAssertEqualObjects(tabline.tablineSelBgColor, [NSColor colorWithRgbInt:0x00ff00]);
+
+    // Test default colors mode
+    // We just verify that the colors changed, rather than asserting the exact
+    // colors to make it easy to update tuning on them in the future.
+    [self setDefault:MMTabColorsModeKey toValue:@(MMTabColorsModeDefaultColors)];
+    [vimView updateTablineColors:MMTabColorsModeDefaultColors];
+
+    vimView.window.appearance = [NSAppearance appearanceNamed: NSAppearanceNameAqua];
+    [self waitForEventHandling];
+    XCTAssertEqual(tabline.tablineFillBgColor.colorSpace.colorSpaceModel, NSColorSpaceModelGray);
+    XCTAssertGreaterThan(tabline.tablineFillBgColor.whiteComponent, 0.5);
+
+    vimView.window.appearance = [NSAppearance appearanceNamed: NSAppearanceNameDarkAqua];
+    [self waitForEventHandling];
+    XCTAssertLessThan(tabline.tablineFillBgColor.whiteComponent, 0.5);
+}
+
 #pragma mark Full screen tests
 
 - (void)waitForNativeFullscreenEnter {

src/MacVim/Miscellaneous.h

@@ -110,6 +110,7 @@ typedef enum : NSInteger {
     MMTabColorsModeDefaultColors = 0,   ///< Use default colors based on macOS light/dark modes
     MMTabColorsModeAutomatic,           ///< Automatically derive tab colors based on foreground/background colors
     MMTabColorsModeVimColorscheme,      ///< Use Vim colorscheme TabLine/TabLineSel/TabLineFill colors
+    MMTabColorsModeCount
 } MMTabColorsMode;
 
 enum {

src/MacVim/gui_macvim.m

@@ -656,32 +656,44 @@
         gui_mch_fuopt_update();
 
     // Update the GUI with tab colors
-    // We can cache the tabline syn IDs because they will never change.
-    static int tablineSynIds[3] = { 0 };
-    char *tablineSynNames[3] = {"TabLine", "TabLineFill", "TabLineSel"};
 
-    BOOL hasTablineColors = YES;
+    // Highlight attributes for TabLine, TabLineFill, TabLineSel
+    const int attrs[3] = { HL_ATTR(HLF_TP), HL_ATTR(HLF_TPF), HL_ATTR(HLF_TPS) };
+
     int tablineColors[6] = { 0 };
     for (int i = 0; i < 3; i++) {
-        if (tablineSynIds[i] <= 0)
-            tablineSynIds[i] = syn_name2id((char_u *)tablineSynNames[i]);
-        if (tablineSynIds[i] > 0) {
-            guicolor_T bg, fg;
-            syn_id2colors(tablineSynIds[i], &fg, &bg);
-            tablineColors[i*2] = (int)bg;
-            tablineColors[i*2+1] = (int)fg;
+        guicolor_T bg = INVALCOLOR, fg = INVALCOLOR;
+        BOOL reverse = NO;
+        if (attrs[i] > HL_ALL) {
+            attrentry_T *aep = syn_gui_attr2entry(attrs[i]);
+            if (aep != NULL) {
+                bg = aep->ae_u.gui.bg_color;
+                fg = aep->ae_u.gui.fg_color;
+                reverse = (aep->ae_attr & HL_INVERSE) != 0;
+            }
         } else {
-            hasTablineColors = NO;
+            reverse = (attrs[i] & HL_INVERSE) != 0;
         }
-    }
-    if (hasTablineColors) {
-        // Cache the old colors just so we don't spam the IPC channel if the
-        // colors didn't actually change.
-        static int oldTablineColors[6] = { 0 };
-        if (memcmp(oldTablineColors, tablineColors, sizeof(oldTablineColors)) != 0) {
-            memcpy(oldTablineColors, tablineColors, sizeof(oldTablineColors));
-            [[MMBackend sharedInstance] setTablineColors:tablineColors];
+
+        if (bg == INVALCOLOR)
+            bg = gui.def_back_pixel;
+        if (fg == INVALCOLOR)
+            fg = gui.def_norm_pixel;
+
+        if (reverse) {
+            guicolor_T temp = fg;
+            fg = bg;
+            bg = temp;
         }
+        tablineColors[i*2] = (int)bg;
+        tablineColors[i*2+1] = (int)fg;
+    }
+    // Cache the old colors just so we don't spam the IPC channel if the
+    // colors didn't actually change.
+    static int oldTablineColors[6] = { 0 };
+    if (memcmp(oldTablineColors, tablineColors, sizeof(oldTablineColors)) != 0) {
+        memcpy(oldTablineColors, tablineColors, sizeof(oldTablineColors));
+        [[MMBackend sharedInstance] setTablineColors:tablineColors];
     }
 }