Fix handling of Unicode chars in RTF generation.

Modified check in WideCharToChar in UEncodings unit to detect failed
conversions from WideChar to a given ANSI code page.

Added support for \uc RTF control word to TRTF in URTFUtils unit.

Fixed problem in TRTF.MakeSafeText that was not handling Unicode
correctly when a Unicode character cannot be represented in a given
code page.

Fixes #157

Also fixed error in detecting ASCII characters that need to be escaped#
in RTF (per issue 159)

Fixes #159

Author: delphidabbler

Src/UEncodings.pas

@@ -437,7 +437,12 @@ function WideCharToChar(const Source: WideChar; const CodePage: Integer;
 var
   UsedDefChar: BOOL;
   BufSize: Integer;
+  Encoding: TEncoding;
+  TestStr: string;
+  TestBytes: TBytes;
+  Idx: Integer;
 begin
+  // Attempt to convert the Unicode char to ANSI char(s)
   BufSize := WideCharToMultiByte(
     CodePage, 0, @Source, 1, nil, 0, nil, nil
   );
@@ -447,7 +452,17 @@ function WideCharToChar(const Source: WideChar; const CodePage: Integer;
   ) = 0 then
     RaiseLastOSError;
   SetLength(Dest, Length(Dest) - 1);
-  Result := not UsedDefChar;
+  // Check if the conversion succeeded
+  Encoding := TMBCSEncoding.Create;
+  try
+    SetLength(TestBytes, Length(Dest));
+    for Idx := 0 to Pred(Length(Dest)) do
+      TestBytes[Idx] := Ord(Dest[Idx]);
+    TestStr := Encoding.GetString(TestBytes);
+    Result := (TestStr = Source) and not UsedDefChar;
+  finally
+    Encoding.Free;
+  end;
 end;
 
 { TEncodingHelper }

Src/URTFUtils.pas

@@ -65,7 +65,8 @@ interface
     Ignore,               // denotes following control can be ignored
     FirstLineOffset,      // first line indent in twips (relative to \li)
     LeftIndent,           // left indent in twips
-    TabStop               // sets a tab stop in twips
+    TabStop,              // sets a tab stop in twips
+    UnicodeCharSize       // number of bytes of a given \uN Unicode character
   );
   {$ScopedEnums off}
 
@@ -81,7 +82,7 @@   TRTF = record
         'fcharset', 'fnil', 'froman', 'fswiss', 'fmodern', 'fscript', 'fdecor',
         'ftech', 'colortbl', 'red', 'green', 'blue', 'info', 'title', 'pard',
         'par', 'plain', 'f', 'cf', 'b', 'i', 'ul', 'fs', 'sb', 'sa', 'u', 'upr',
-        'ud', '*', 'fi', 'li', 'tx'
+        'ud', '*', 'fi', 'li', 'tx', 'uc'
       );
   strict private
 
@@ -134,13 +135,13 @@   TRTF = record
     ///  </returns>
     ///  <remarks>Converted characters are escaped if necessary. Any characters
     ///  that are not valid in the required code page are encoded in a Unicode
-    ///  RTF control word with <c>?</c> as the non-Unicode fallback.</remarks>
+    ///  RTF control word with a non-Unicode fallback.</remarks>
     class function MakeSafeText(const AText: string; const ACodePage: Integer):
       ASCIIString; static;
 
     ///  <summary>Creates an RTF destination in a Unicode safe way.</summary>
     ///  <param name="ADestCtrl"><c>TRTFControl</c> [in] Required destination
-    ///   control.</param>
+    ///  control.</param>
     ///  <param name="ADestText"><c>string</c> [in] Unicode text to be included
     ///  in the destination.</param>
     ///  <param name="ACodePage"><c>Integer</c> [in] ANSI Code page to use for
@@ -152,8 +153,7 @@   TRTF = record
     ///  containing the encoded text, escaped as necessary. Should any
     ///  characters in <c>ADestText</c> be incompatible with the code page then
     ///  two sub-destinations are created, one containing Unicode characters and
-    ///  the other containing ANSI text, with unknown characters flagged with
-    ///  &quot;error&quot; characters such as <c>?</c>.</remarks>
+    ///  the other containing ANSI text.</remarks>
     class function UnicodeSafeDestination(const ADestCtrl: TRTFControl;
       const ADestText: string; const ACodePage: Integer): ASCIIString; static;
   end;
@@ -250,6 +250,20 @@ class function TRTF.HexEscape(const Ch: AnsiChar): ASCIIString;
 
 class function TRTF.MakeSafeText(const AText: string; const ACodePage: Integer):
   ASCIIString;
+
+  function MakeSafeChar(const AChar: AnsiChar): ASCIIString;
+  begin
+    if (AChar < #$20) or ((AChar >= #$7F) and (AChar <= #$FF)) then
+      // Not an ASCII character
+      Result := HexEscape(AChar)
+    else if (AChar = '{') or (AChar = '\') or (AChar = '}') then
+      // Reserved RTF character: must be escaped
+      Result := Escape(AChar)
+    else
+      // Valid character, use as is
+      Result := ASCIIString(AChar);
+  end;
+
 var
   Ch: Char;                     // each Unicode character in TheText
   AnsiChars: TArray<AnsiChar>;  // translation of a Ch into the ANSI code page
@@ -264,23 +278,23 @@ class function TRTF.MakeSafeText(const AText: string; const ACodePage: Integer):
     begin
       // Conversion succeeded: check process each ANSI char
       for AnsiCh in AnsiChars do
-      begin
-        if (AnsiCh < #$20) or ((AnsiCh >= #$7F) and (AnsiCh <= #$FF)) then
-          // Not an ASCII character
-          Result := Result + HexEscape(AnsiCh)
-        else if (Ch = '{') or (Ch = '\') or (Ch = '}') then
-          // Reserved RTF character: must be escaped
-          Result := Result + Escape(AnsiCh)
-        else
-          // Valid character, use as is
-          Result := Result + ASCIIString(AnsiCh);
-      end;
+        Result := Result + MakeSafeChar(AnsiCh)
     end
     else
-      // Conversion failed: we store Unicode char in a Unicode control word
+    begin
+      // Conversion failed: create a Unicode character followed by fallback
+      // ANSI character
       Result := Result
+        + ControlWord(TRTFControl.UnicodeCharSize, 1)
         + ControlWord(TRTFControl.UnicodeChar, SmallInt(Ord(Ch)))
-        + ' ?';   // fallback "unprintable" value
+        + ' ';
+      if Length(AnsiChars) = 1 then
+        // Single alternate character: output it
+        Result := Result + MakeSafeChar(AnsiChars[0])
+      else
+        // Can't get alternate: use '?'
+        Result := Result + '?';
+    end;
   end;
 end;