Make sure the inverted cursor is always readable (#3647)

.github/actions/spelling/allow/apis.txt

@@ -103,6 +103,7 @@ lround
 Lsa
 lsass
 LSHIFT
+LTGRAY
 MAINWINDOW
 memchr
 memicmp
@@ -146,6 +147,7 @@ OUTLINETEXTMETRICW
 overridable
 PACL
 PAGESCROLL
+PATINVERT
 PEXPLICIT
 PICKFOLDERS
 pmr

src/renderer/atlas/shader_ps.hlsl

@@ -101,13 +101,27 @@ float4 main(float4 pos: SV_Position): SV_Target
     // Colored cursors are drawn "in between" the background color and the text of a cell.
     [branch] if (cell.flags & CellFlags_Cursor)
     {
-        [flatten] if (cursorColor != INVALID_COLOR)
+        [branch] if (cursorColor != INVALID_COLOR)
         {
             // The cursor texture is stored at the top-left-most glyph cell.
             // Cursor pixels are either entirely transparent or opaque.
             // --> We can just use .a as a mask to flip cursor pixels on or off.
             color = alphaBlendPremultiplied(color, decodeRGBA(cursorColor) * glyphs[cellPos].a);
         }
+        else if (glyphs[cellPos].a != 0)
+        {
+            // Make sure the cursor is always readable (see gh-3647)
+            // If we imagine the two colors to be in 0-255 instead of 0-1,
+            // this effectively XORs them with 63. This avoids a situation
+            // where a gray background color (0.5) gets inverted to the
+            // same gray making the cursor invisible.
+            float2x4 colors = { color, fg };
+            float2x4 ip; // integral part
+            float2x4 frac = modf(colors * (255.0f / 64.0f), ip);
+            colors = (3.0f - ip + frac) * (64.0f / 255.0f);
+            color = float4(colors[0].rgb, 1);
+            fg = float4(colors[1].rgb, 1);
+        }
     }
 
     // Layer 2:
@@ -146,7 +160,7 @@ float4 main(float4 pos: SV_Position): SV_Target
         // What a nice coincidence that we have exactly 8 flags to handle right now!
         // `mask` will mask away any positive results from checks we don't want.
         // (I.e. even if we're in an underline, it doesn't matter if we don't want an underline.)
-        bool4x2 mask = {
+        bool2x4 mask = {
             cell.flags & CellFlags_BorderLeft,
             cell.flags & CellFlags_BorderTop,
             cell.flags & CellFlags_BorderRight,
@@ -159,7 +173,7 @@ float4 main(float4 pos: SV_Position): SV_Target
         // The following <lineWidth checks rely on underflow turning the
         // uint into a way larger number than any reasonable lineWidth.
         // That way we don't need to write `y >= lo && y < hi`.
-        bool4x2 checks = {
+        bool2x4 checks = {
             // These 2 expand to 4 bools, because cellPos is a
             // uint2 vector which results in a bool2 result each.
             cellPos < thinLineWidth,
@@ -176,16 +190,6 @@ float4 main(float4 pos: SV_Position): SV_Target
         }
     }
 
-    // Layer 3 (optional):
-    // Uncolored cursors are used as a mask that inverts the cells color.
-    [branch] if (cell.flags & CellFlags_Cursor)
-    {
-        [flatten] if (cursorColor == INVALID_COLOR && glyphs[cellPos].a != 0)
-        {
-            color = float4(1 - color.rgb, 1);
-        }
-    }
-
     // Layer 4:
     // The current selection is drawn semi-transparent on top.
     [branch] if (cell.flags & CellFlags_Selected)

src/renderer/dx/CustomTextRenderer.cpp

@@ -395,8 +395,8 @@ try
         if (firstPass)
         {
             // Draw a backplate behind the cursor in the *background* color so that we can invert it later.
-            // We're going to draw the exact same color as the background behind the cursor
-            const til::color color{ drawingContext.backgroundBrush->GetColor() };
+            // Make sure the cursor is always readable (see gh-3647)
+            const til::color color{ til::color{ drawingContext.backgroundBrush->GetColor() } ^ RGB(63, 63, 63) };
             RETURN_IF_FAILED(d2dContext->CreateSolidColorBrush(color.with_alpha(255),
                                                                &brush));
         }

src/renderer/gdi/paint.cpp

@@ -733,7 +733,11 @@ bool GdiEngine::FontHasWesternScript(HDC hdc)
 
         for (auto r : cursorInvertRects)
         {
-            RETURN_HR_IF(E_FAIL, !(InvertRect(_hdcMemoryContext, &r)));
+            // Make sure the cursor is always readable (see gh-3647)
+            const auto PrevObject = SelectObject(_hdcMemoryContext, GetStockObject(LTGRAY_BRUSH));
+            const auto Result = PatBlt(_hdcMemoryContext, r.left, r.top, r.right - r.left, r.bottom - r.top, PATINVERT);
+            SelectObject(_hdcMemoryContext, PrevObject);
+            RETURN_HR_IF(E_FAIL, !Result);
         }
     }