sf.net # 795 : unsupported \UNNNN escape sequence is miscompiled

- escape codepoints > 0xFFFF with "\Unnnnnnnn"
- preserve "\Unnnnnnnn" escape sequences where code point > 0xFFFF
- expect "\Unnnnnnnn" escape to have 8 hex digits
- warn when "\u" or "\U" escape sequences  are invalid or truncated
- add capability to return a warning from hReEscape() and hReEscapeW()
- internal - order of functions in src/compiler/hlp-str.bas
- "\Unnnnnnnn" escape sequence is mapped as follows:
  when within unicode BMP, "\unnnn", otherwise on linux "\Unnnnnnnn",
  or on windows "\uD8XX\uDCXX" surrogate pair.

Author: jayrm

changelog.txt

@@ -26,6 +26,8 @@ Version 1.20.0
 - fbc: set bit value '__FB_ERR__' = &h400 if '-earraydims' is enabled
 - PEEK, POKE, and SWAP are now quirk words instead of keywords allowing these names to be used as member procedure names
 - fbdoc: change CRegex.bas to use pcre2 since the libpcre (a.k.a pcre3) is obsolete in favour of pcre2 by upstream (Ahmad Khalifa)
+- PEEK, POKE, and SWAP are now quirk words instead of keywords allowing these names to be used as member procedure names
+- "\Unnnnnnnn" escape sequence is mapped as follows: when within unicode BMP, "\unnnn", otherwise on linux "\Unnnnnnnn", or on windows "\uD8XX\uDCXX" surrogate pair.  
 
 [added]
 - x86_64: optimize SHL MOD INTDIV to use 32-bit operation when result will be converted to long/ulong
@@ -66,6 +68,7 @@ Version 1.20.0
 - '-earraydims' command line option to enable array dimensions checking.  Enabled by default with '-exx' command line option.
 - github #426: add __FB_ARG_LISTEXPAND__( macroname, macroargcount, args... ): expands to one or more 'macroname( .... )' depending on the value of macroargcount and number of arguments in the args... list (skyfish)
 - fbc: command line option '-z optabstract' to support optimizing purely abstract types, and only preserve used pure abstract types (skyfish)
+- fbc: warning when "\u" or "\U" escape sequences are invalid or truncated
 
 [fixed]
 - github #410: give consistent floating point comparisons results involving NaNs.
@@ -100,7 +103,8 @@ Version 1.20.0
 - gas64: fix casting ulong (with "negative value") to double/single (SARG)
 - gas64: fix optimization prevented converting 32bit memory/register (SARG)
 - fbc: #cmdline "-end" was not resetting the parser, hiding errors and producing a bad executable
-- sf.net #1009: Invalid symbol when accessing indexed PROPERTY arrays BYREF 
+- sf.net #1009: Invalid symbol when accessing indexed PROPERTY arrays BYREF
+- sf.net #795: unsupported \UNNNN escape sequence is miscompiled - expect "\Unnnnnnnn" escape to have 8 digits  
 
 
 Version 1.10.3

src/compiler/error.bas

@@ -94,6 +94,8 @@ declare function hMakeParamDesc _
 		( /'FB_WARNINGMSG_UPCASTDISCARDSINITIALIZER '/ 1, @"Up-casting discards initializer(s)" ), _
 		( /'FB_WARNINGMSG_BYREFTEMPVAR              '/ 2, @"Suspicious address expression passed to BYREF parameter" ), _
 		( /'FB_WARNINGMSG_MISSINGANDROIDSYSROOT     '/ 1, @"GCC/Clang didn't provide a proper sysroot. You probably have to pass fbc an argument of the form '-sysroot $NDK/platforms/android-$API/arch-$ARCH'" ), _
+		( /'FB_WARNINGMSG_INVALIDUNICODESEQUENCE    '/ 2, @"Invalid or truncated unicode escape sequence" ), _
+		( /'FB_WARNINGMSG_SURROGATEPAIR             '/ 1, @"Surrogate or surrogate pair in unicode literal" ), _
 		( /'FB_WARNINGMSGS                          '/ 0, @"FB_WARNINGMSGS" ) _
 	}
 

src/compiler/error.bi

@@ -392,6 +392,8 @@ enum FB_WARNINGMSG
 	FB_WARNINGMSG_UPCASTDISCARDSINITIALIZER
 	FB_WARNINGMSG_BYREFTEMPVAR
 	FB_WARNINGMSG_MISSINGANDROIDSYSROOT
+	FB_WARNINGMSG_INVALIDUNICODESEQUENCE
+	FB_WARNINGMSG_SURROGATEPAIR
 
 	FB_WARNINGMSGS
 end enum

src/compiler/hash.bas

@@ -3,6 +3,7 @@
 '' chng: sep/2004 written [v1ctor]
 ''       jan/2005 updated to use real linked-lists [v1ctor]
 
+#include once "fb.bi"
 #include once "hash.bi"
 #include once "hlp.bi"