Fixes #4387. Runes should not be used on a cell, but rather should use a single grapheme rendering 1 or 2 columns

Directory.Packages.props

@@ -18,7 +18,7 @@
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="[9.0.0,10)" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.6" />
     <PackageVersion Include="System.IO.Abstractions" Version="[22.0.16,23)" />
-    <PackageVersion Include="Wcwidth" Version="[3.0.0,)" />
+    <PackageVersion Include="Wcwidth" Version="[4.0.0,)" />
     <PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="[1.21.2,2)" />
     <PackageVersion Include="Serilog" Version="4.2.0" />
     <PackageVersion Include="Serilog.Extensions.Logging" Version="9.0.0" />

Examples/UICatalog/Scenarios/CombiningMarks.cs

@@ -16,7 +16,8 @@ public override void Main ()
             Application.Current!.SetNeedsDraw ();
 
             var i = -1;
-            top.AddStr ("Terminal.Gui only supports combining marks that normalize. See Issue #2616.");
+            top.Move (0, ++i);
+            top.AddStr ("Terminal.Gui supports all combining sequences that can be rendered as an unique grapheme.");
             top.Move (0, ++i);
             top.AddStr ("\u0301<- \"\\u0301\" using AddStr.");
             top.Move (0, ++i);
@@ -38,7 +39,7 @@ public override void Main ()
             top.AddRune ('\u0301');
             top.AddRune ('\u0328');
             top.AddRune (']');
-            top.AddStr ("<- \"[a\\u0301\\u0301\\u0328]\" using AddRune for each.");
+            top.AddStr ("<- \"[a\\u0301\\u0301\\u0328]\" using AddRune for each. Avoid use AddRune for combining sequences because may result with empty blocks at end.");
             top.Move (0, ++i);
             top.AddStr ("[a\u0301\u0301\u0328]<- \"[a\\u0301\\u0301\\u0328]\" using AddStr.");
             top.Move (0, ++i);
@@ -82,6 +83,16 @@ public override void Main ()
             top.AddStr ("[\U0001F468\U0001F469\U0001F9D2]<- \"[\\U0001F468\\U0001F469\\U0001F9D2]\" using AddStr.");
             top.Move (0, ++i);
             top.AddStr ("[\U0001F468\u200D\U0001F469\u200D\U0001F9D2]<- \"[\\U0001F468\\u200D\\U0001F469\\u200D\\U0001F9D2]\" using AddStr.");
+            top.Move (0, ++i);
+            top.AddStr ("[\U0001F468\u200D\U0001F469\u200D\U0001F467\u200D\U0001F466]<- \"[\\U0001F468\\u200D\\U0001F469\\u200D\\U0001F467\\u200D\\U0001F466]\" using AddStr.");
+            top.Move (0, ++i);
+            top.AddStr ("[\u0e32\u0e33]<- \"[\\u0e32\\u0e33]\" using AddStr.");
+            top.Move (0, ++i);
+            top.AddStr ("[\U0001F469\u200D\u2764\uFE0F\u200D\U0001F48B\u200D\U0001F468]<- \"[\\U0001F469\\u200D\\u2764\\uFE0F\\u200D\\U0001F48B\\u200D\\U0001F468]\" using AddStr.");
+            top.Move (0, ++i);
+            top.AddStr ("[\u0061\uFE20\u0065\uFE21]<- \"[\\u0061\\uFE20\\u0065\\uFE21]\" using AddStr.");
+            top.Move (0, ++i);
+            top.AddStr ("[\u1100\uD7B0]<- \"[\\u1100\\uD7B0]\" using AddStr.");
         };
 
         Application.Run (top);

Examples/UICatalog/Scenarios/LineDrawing.cs

@@ -284,7 +284,7 @@ protected override bool OnDrawingContent ()
                     SetCurrentAttribute (c.Value.Value.Attribute ?? GetAttributeForRole (VisualRole.Normal));
 
                     // TODO: #2616 - Support combining sequences that don't normalize
-                    AddRune (c.Key.X, c.Key.Y, c.Value.Value.Rune);
+                    AddStr (c.Key.X, c.Key.Y, c.Value.Value.Grapheme);
                 }
             }
         }

Examples/UICatalog/Scenarios/Sliders.cs

@@ -86,17 +86,17 @@ public void MakeSliders (View v, List<object> options)
                                 {
                                     if (single.Orientation == Orientation.Horizontal)
                                     {
-                                        single.Style.SpaceChar = new () { Rune = Glyphs.HLine };
-                                        single.Style.OptionChar = new () { Rune = Glyphs.HLine };
+                                        single.Style.SpaceChar = new () { Grapheme = Glyphs.HLine.ToString () };
+                                        single.Style.OptionChar = new () { Grapheme = Glyphs.HLine.ToString () };
                                     }
                                     else
                                     {
-                                        single.Style.SpaceChar = new () { Rune = Glyphs.VLine };
-                                        single.Style.OptionChar = new () { Rune = Glyphs.VLine };
+                                        single.Style.SpaceChar = new () { Grapheme = Glyphs.VLine.ToString () };
+                                        single.Style.OptionChar = new () { Grapheme = Glyphs.VLine.ToString () };
                                     }
                                 };
-        single.Style.SetChar = new () { Rune = Glyphs.ContinuousMeterSegment };
-        single.Style.DragChar = new () { Rune = Glyphs.ContinuousMeterSegment };
+        single.Style.SetChar = new () { Grapheme = Glyphs.ContinuousMeterSegment.ToString () };
+        single.Style.DragChar = new () { Grapheme = Glyphs.ContinuousMeterSegment.ToString () };
 
         v.Add (single);
 
@@ -257,7 +257,7 @@ public override void Main ()
                                                     {
                                                         s.Orientation = Orientation.Horizontal;
 
-                                                        s.Style.SpaceChar = new () { Rune = Glyphs.HLine };
+                                                        s.Style.SpaceChar = new () { Grapheme = Glyphs.HLine.ToString () };
 
                                                         if (prev == null)
                                                         {
@@ -275,7 +275,7 @@ public override void Main ()
                                                     {
                                                         s.Orientation = Orientation.Vertical;
 
-                                                        s.Style.SpaceChar = new () { Rune = Glyphs.VLine };
+                                                        s.Style.SpaceChar = new () { Grapheme = Glyphs.VLine.ToString () };
 
                                                         if (prev == null)
                                                         {

Examples/UICatalog/Scenarios/SyntaxHighlighting.cs

@@ -152,12 +152,12 @@ public override void Main ()
                              ),
                          null,
                          new (
-                              "_Load Rune Cells",
+                              "_Load Text Cells",
                               "",
                               () => ApplyLoadCells ()
                              ),
                          new (
-                              "_Save Rune Cells",
+                              "_Save Text Cells",
                               "",
                               () => SaveCells ()
                              ),
@@ -240,12 +240,9 @@ private void ApplyLoadCells ()
         {
             string csName = color.Key;
 
-            foreach (Rune rune in csName.EnumerateRunes ())
-            {
-                cells.Add (new () { Rune = rune, Attribute = color.Value.Normal });
-            }
+            cells.AddRange (Cell.ToCellList (csName, color.Value.Normal));
 
-            cells.Add (new () { Rune = (Rune)'\n', Attribute = color.Value.Focus });
+            cells.Add (new () { Grapheme = "\n", Attribute = color.Value.Focus });
         }
 
         if (File.Exists (_path))

Terminal.Gui/Drawing/Cell.cs

@@ -1,93 +1,117 @@
 
-
 namespace Terminal.Gui.Drawing;
 
 /// <summary>
 ///     Represents a single row/column in a Terminal.Gui rendering surface (e.g. <see cref="LineCanvas"/> and
 ///     <see cref="IDriver"/>).
 /// </summary>
-public record struct Cell (Attribute? Attribute = null, bool IsDirty = false, Rune Rune = default)
+public record struct Cell (Attribute? Attribute = null, bool IsDirty = false, string Grapheme = "")
 {
     /// <summary>The attributes to use when drawing the Glyph.</summary>
     public Attribute? Attribute { get; set; } = Attribute;
 
     /// <summary>
-    ///     Gets or sets a value indicating whether this <see cref="T:Terminal.Gui.Cell"/> has been modified since the
+    ///     Gets or sets a value indicating whether this <see cref="T:Terminal.Gui.Drawing.Cell"/> has been modified since the
     ///     last time it was drawn.
     /// </summary>
     public bool IsDirty { get; set; } = IsDirty;
 
-    private Rune _rune = Rune;
+    private string _grapheme = Grapheme;
 
-    /// <summary>The character to display. If <see cref="Rune"/> is <see langword="null"/>, then <see cref="Rune"/> is ignored.</summary>
-    public Rune Rune
+    /// <summary>
+    ///     The single grapheme cluster to display from this cell. If <see cref="Grapheme"/> is <see langword="null"/> or
+    ///     <see cref="string.Empty"/>, then <see cref="Cell"/> is ignored.
+    /// </summary>
+    public string Grapheme
     {
-        get => _rune;
+        readonly get => _grapheme;
         set
         {
-            _combiningMarks?.Clear ();
-            _rune = value;
-        }
-    }
+            if (GraphemeHelper.GetGraphemes(value).ToArray().Length > 1)
+            {
+                throw new InvalidOperationException ($"Only a single {nameof (Grapheme)} cluster is allowed per Cell.");
+            }
 
-    private List<Rune>? _combiningMarks;
+            if (!string.IsNullOrEmpty (value) && value.Length == 1 && char.IsSurrogate (value [0]))
+            {
+                throw new ArgumentException ($"Only valid Unicode scalar values are allowed in a single {nameof (Grapheme)} cluster.");
+            }
 
-    /// <summary>
-    ///     The combining marks for <see cref="Rune"/> that when combined makes this Cell a combining sequence. If
-    ///     <see cref="CombiningMarks"/> empty, then <see cref="CombiningMarks"/> is ignored.
-    /// </summary>
-    /// <remarks>
-    ///     Only valid in the rare case where <see cref="Rune"/> is a combining sequence that could not be normalized to a
-    ///     single Rune.
-    /// </remarks>
-    internal IReadOnlyList<Rune> CombiningMarks
-    {
-        // PERFORMANCE: Downside of the interface return type is that List<T> struct enumerator cannot be utilized, i.e. enumerator is allocated.
-        // If enumeration is used heavily in the future then might be better to expose the List<T> Enumerator directly via separate mechanism.
-        get
-        {
-            // Avoid unnecessary list allocation.
-            if (_combiningMarks == null)
+            try
             {
-                return Array.Empty<Rune> ();
+                _grapheme = !string.IsNullOrEmpty (value) && !value.IsNormalized (NormalizationForm.FormC)
+                                ? value.Normalize (NormalizationForm.FormC)
+                                : value;
+            }
+            catch (ArgumentException)
+            {
+                // leave text unnormalized
+                _grapheme = value;
             }
-            return _combiningMarks;
         }
     }
 
     /// <summary>
-    ///     Adds combining mark to the cell.
+    ///     The rune for <see cref="Grapheme"/> or runes for <see cref="Grapheme"/> that when combined makes this Cell a combining sequence.
     /// </summary>
-    /// <param name="combiningMark">The combining mark to add to the cell.</param>
-    internal void AddCombiningMark (Rune combiningMark)
+    /// <remarks>
+    ///     In the case where <see cref="Grapheme"/> has more than one rune it is a combining sequence that is normalized to a
+    ///     single Text which may occupies 1 or 2 columns.
+    /// </remarks>
+    public IReadOnlyList<Rune> Runes => string.IsNullOrEmpty (Grapheme) ? [] : Grapheme.EnumerateRunes ().ToList ();
+
+    /// <inheritdoc/>
+    public override string ToString ()
     {
-        _combiningMarks ??= [];
-        _combiningMarks.Add (combiningMark);
+        string visibleText = EscapeControlAndInvisible (Grapheme);
+
+        return $"[\"{visibleText}\":{Attribute}]";
     }
 
-    /// <summary>
-    ///     Clears combining marks of the cell.
-    /// </summary>
-    internal void ClearCombiningMarks ()
+    private static string EscapeControlAndInvisible (string text)
     {
-        _combiningMarks?.Clear ();
-    }
+        if (string.IsNullOrEmpty (text))
+        {
+            return "";
+        }
 
-    /// <inheritdoc/>
-    public override string ToString () { return $"['{Rune}':{Attribute}]"; }
+        var sb = new StringBuilder ();
+
+        foreach (var rune in text.EnumerateRunes ())
+        {
+            switch (rune.Value)
+            {
+                case '\0': sb.Append ("␀"); break;
+                case '\t': sb.Append ("\\t"); break;
+                case '\r': sb.Append ("\\r"); break;
+                case '\n': sb.Append ("\\n"); break;
+                case '\f': sb.Append ("\\f"); break;
+                case '\v': sb.Append ("\\v"); break;
+                default:
+                    if (char.IsControl ((char)rune.Value))
+                    {
+                        // show as \uXXXX
+                        sb.Append ($"\\u{rune.Value:X4}");
+                    }
+                    else
+                    {
+                        sb.Append (rune);
+                    }
+                    break;
+            }
+        }
+
+        return sb.ToString ();
+    }
 
     /// <summary>Converts the string into a <see cref="List{Cell}"/>.</summary>
     /// <param name="str">The string to convert.</param>
     /// <param name="attribute">The <see cref="Scheme"/> to use.</param>
     /// <returns></returns>
     public static List<Cell> ToCellList (string str, Attribute? attribute = null)
     {
-        List<Cell> cells = new ();
-
-        foreach (Rune rune in str.EnumerateRunes ())
-        {
-            cells.Add (new () { Rune = rune, Attribute = attribute });
-        }
+        List<Cell> cells = [];
+        cells.AddRange (GraphemeHelper.GetGraphemes (str).Select (grapheme => new Cell { Grapheme = grapheme, Attribute = attribute }));
 
         return cells;
     }
@@ -100,9 +124,7 @@ public static List<Cell> ToCellList (string str, Attribute? attribute = null)
     /// <returns>A <see cref="List{Cell}"/> for each line.</returns>
     public static List<List<Cell>> StringToLinesOfCells (string content, Attribute? attribute = null)
     {
-        List<Cell> cells = content.EnumerateRunes ()
-                                  .Select (x => new Cell { Rune = x, Attribute = attribute })
-                                  .ToList ();
+        List<Cell> cells = ToCellList (content, attribute);
 
         return SplitNewLines (cells);
     }
@@ -112,14 +134,14 @@ public static List<List<Cell>> StringToLinesOfCells (string content, Attribute?
     /// <returns></returns>
     public static string ToString (IEnumerable<Cell> cells)
     {
-        var str = string.Empty;
+        StringBuilder sb = new ();
 
         foreach (Cell cell in cells)
         {
-            str += cell.Rune.ToString ();
+            sb.Append (cell.Grapheme);
         }
 
-        return str;
+        return sb.ToString ();
     }
 
     /// <summary>Converts a <see cref="List{Cell}"/> generic collection into a string.</summary>
@@ -147,26 +169,19 @@ public static string ToString (List<List<Cell>> cellsList)
 
     internal static List<Cell> StringToCells (string str, Attribute? attribute = null)
     {
-        List<Cell> cells = [];
-
-        foreach (Rune rune in str.ToRunes ())
-        {
-            cells.Add (new () { Rune = rune, Attribute = attribute });
-        }
-
-        return cells;
+        return ToCellList (str, attribute);
     }
 
-    internal static List<Cell> ToCells (IEnumerable<Rune> runes, Attribute? attribute = null)
+    internal static List<Cell> ToCells (IEnumerable<string> strings, Attribute? attribute = null)
     {
-        List<Cell> cells = new ();
+        StringBuilder sb = new ();
 
-        foreach (Rune rune in runes)
+        foreach (string str in strings)
         {
-            cells.Add (new () { Rune = rune, Attribute = attribute });
+            sb.Append (str);
         }
 
-        return cells;
+        return ToCellList (sb.ToString (), attribute);
     }
 
     private static List<List<Cell>> SplitNewLines (List<Cell> cells)
@@ -179,14 +194,15 @@ private static List<List<Cell>> SplitNewLines (List<Cell> cells)
         // ASCII code 10 = Line Feed.
         for (; i < cells.Count; i++)
         {
-            if (cells [i].Rune.Value == 13)
+            if (cells [i].Grapheme.Length == 1 && cells [i].Grapheme [0] == 13)
             {
                 hasCR = true;
 
                 continue;
             }
 
-            if (cells [i].Rune.Value == 10)
+            if ((cells [i].Grapheme.Length == 1 && cells [i].Grapheme [0] == 10)
+                || cells [i].Grapheme == "\r\n")
             {
                 if (i - start > 0)
                 {