Add NormalizeUTF8 utility function

Author: facchinm

src/arduino.cc/builder/utils/utils.go

@@ -40,8 +40,12 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
+
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/gohasissues"
 	"arduino.cc/builder/i18n"
@@ -535,6 +539,18 @@ func ParseCppString(line string) (string, string, bool) {
 	}
 }
 
+func isMn(r rune) bool {
+	return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
+}
+
+// Normalizes an UTF8 byte slice
+// TODO: use it more often troughout all the project (maybe on logger interface?)
+func NormalizeUTF8(buf []byte) []byte {
+	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
+	result, _, _ := transform.Bytes(t, buf)
+	return result
+}
+
 // CopyFile copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents