Improve RGB→256-colour palette interpolation by trying gray and cube

The 6×6×6 colour cube includes gray colours.  Black and white are
already used but the cube also includes other grays such as #5f5f5f
or #d7d7d7.  For those colours it’s better to use colour from the
cube rather than from the grayscale ramp.

On the flip side, it is better to approximate #080807 by #080808
present in te grayscale ramp rather than black from the colour cube.

Change to code to try both approximation and choose whichever has
in smaller distance to the original colour.

src/terminal.rs

@@ -4,41 +4,62 @@ use syntect::highlighting::{self, FontStyle};
 
 /// Approximate a 24-bit color value by a 8 bit ANSI code
 fn rgb2ansi(r: u8, g: u8, b: u8) -> u8 {
-    if r == g && g == b {
-        rgb2ansi_grey(r)
+    let (grey_index, grey_distance) = rgb2ansi_grey(r, g, b);
+    if grey_distance == 0 {
+        return grey_index;
+    }
+    let (cube_index, cube_distance) = rgb2ansi_cube(r, g, b);
+    if grey_distance < cube_distance {
+        grey_index
     } else {
-        rgb2ansi_cube(r, g, b)
+        cube_index
     }
 }
 
 /// Approximate a 24-bit colour as an index in greyscale ramp of the 256-colour
 /// ANSI palette.
 #[inline]
-fn rgb2ansi_grey(y: u8) -> u8 {
+fn rgb2ansi_grey(r: u8, g: u8, b: u8) -> (u8, u32) {
     const BLACK: u8 = 16;
     const WHITE: u8 = 231;
 
+    let y = if r == g && g == b {
+        r
+    } else {
+        // Technically we should convert RGB to linear RGB but that’s too much
+        // work.  We would need to go to floating points, use power formula and
+        // in the end no one will notice all the hard work we’ve put into it!
+        // Instead just cheat and use gamma-corrected RGB values.  Coefficients
+        // from <https://www.w3.org/TR/WCAG20-TECHS/G17.html#G17-tests>.
+        (((r as u32) * 2126 + (g as u32) * 7152 + (b as u32) * 722) / 10000) as u8
+    };
+
     // The greyscale ramp starts at rgb(8, 8, 8) and steps every rgb(10, 10, 10)
     // until rgb(238, 238, 238).  We’re adding 6 so that division rounds to
     // nearest rather than truncating.  Due to asymmetry at the edges those need
     // to be handled specially.
     if y < 4 {
-        BLACK
+        (BLACK, distance(r, g, b, 0, 0, 0))
     } else if y >= 247 {
-        WHITE
+        (WHITE, distance(r, g, b, 255, 255, 255))
     } else {
-        231 + if y >= 234 { 24 } else { ((y + 6) / 10) as u8 }
+        let i = if y >= 234 { 24 } else { (y + 6) / 10 };
+        let y = i * 10 - 2;
+        (i + 231, distance(r, g, b, y, y, y))
     }
 }
 
 /// Approximate a 24-bit colour as an index in 6×6×6 colour cube of the
 /// 256-colour ANSI palette.
 #[inline]
-fn rgb2ansi_cube(r: u8, g: u8, b: u8) -> u8 {
+fn rgb2ansi_cube(r: u8, g: u8, b: u8) -> (u8, u32) {
     let ri = cube_index(r);
     let gi = cube_index(g);
     let bi = cube_index(b);
-    16 + ri * 36 + gi * 6 + bi
+    (
+        16 + ri * 36 + gi * 6 + bi,
+        distance(r, g, b, cube_value(ri), cube_value(gi), cube_value(bi)),
+    )
 }
 
 /// Approximates single r, g or b value to an index within a single side of the
@@ -58,7 +79,6 @@ fn cube_index(v: u8) -> u8 {
 
 /// Converts index on one dimension of the 6×6×6 ANSI colour cube into value in
 /// sRGB space.
-#[cfg(test)]
 fn cube_value(i: u8) -> u8 {
     if i == 0 {
         0
@@ -71,7 +91,6 @@ fn cube_value(i: u8) -> u8 {
 /// computation and perceptual correctness.  Returned value is not a proper
 /// metric but two properties this gives us are: d(x, x) = 0 and d(x, y) < d(x,
 /// z) implies y is closer to x than z is to x.
-#[cfg(test)]
 fn distance(r1: u8, g1: u8, b1: u8, r2: u8, g2: u8, b2: u8) -> u32 {
     let r_mean = (r1 as i32 + r2 as i32) / 2;
     let r = r1 as i32 - r2 as i32;
@@ -134,11 +153,23 @@ fn test_rgb2ansi_color() {
     assert_eq!(193, rgb2ansi(0xd7, 0xff, 0xaf));
 }
 
+#[test]
+fn test_rgb2ansi_grey_using_cube() {
+    // Even though those are grey colours they have a perfect match in the
+    // colour cube so use that rather than greyscale ramp.
+    assert_eq!(59, rgb2ansi(0x5f, 0x5f, 0x5f));
+    assert_eq!(102, rgb2ansi(0x87, 0x87, 0x87));
+    assert_eq!(145, rgb2ansi(0xaf, 0xaf, 0xaf));
+}
+
 #[test]
 fn test_rgb2ansi_approx() {
     assert_eq!(231, rgb2ansi(0xfe, 0xfe, 0xfe));
     // Approximate #070707 up to #080808 rather than down to #000000.
     assert_eq!(232, rgb2ansi(0x07, 0x07, 0x07));
+    // Even though #080708 is not a grey colour, approximation to #080808 in
+    // greyscale ramp is better than to #000000 in the colour cube.
+    assert_eq!(232, rgb2ansi(0x08, 0x07, 0x08));
 }
 
 #[test]
@@ -177,6 +208,6 @@ fn test_distance() {
     }
 
     let avg_distance = total_distance / 16777216.;
-    assert_eq!(47, max_distance.round() as u32);
+    assert_eq!(41, max_distance.round() as u32);
     assert_eq!(17, avg_distance.round() as u32);
 }