searchKeys and GetString now handle escaped strings

Author: daboyuka

bytes_safe.go

@@ -17,5 +17,5 @@ func parseFloat(b *[]byte) (float64, error) {
 }
 
 func bytesToString(b *[]byte) string {
-  return string(*b)
-}
+	return string(*b)
+}

bytes_unsafe.go

@@ -27,5 +27,5 @@ func parseFloat(b *[]byte) (float64, error) {
 // A hack until issue golang/go#2632 is fixed.
 // See: https://github.com/golang/go/issues/2632
 func bytesToString(b *[]byte) string {
-  return *(*string)(unsafe.Pointer(b))
-}
+	return *(*string)(unsafe.Pointer(b))
+}

parser.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"strconv"
 )
 
 // Errors
@@ -19,6 +18,10 @@ var (
 	MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string")
 )
 
+// How much stack space to allocate for unescaping JSON strings; if a string longer
+// than this needs to be escaped, it will result in a heap allocation
+const unescapeStackBufSize = 64
+
 func tokenEnd(data []byte) int {
 	for i, c := range data {
 		switch c {
@@ -46,24 +49,32 @@ func nextToken(data []byte) int {
 
 // Tries to find the end of string
 // Support if string contains escaped quote symbols.
-func stringEnd(data []byte) int {
+func stringEnd(data []byte) (int, bool) {
+	escaped := false
 	for i, c := range data {
 		if c == '"' {
-			j := i - 1
-			for {
-				if j < 0 || data[j] != '\\' {
-					return i + 1 // even number of backslashes
-				}
-				j--
-				if j < 0 || data[j] != '\\' {
-					break // odd number of backslashes
+			if !escaped {
+				return i + 1, false
+			} else {
+				j := i - 1
+				for {
+					if j < 0 || data[j] != '\\' {
+						return i + 1, true // even number of backslashes
+					}
+					j--
+					if j < 0 || data[j] != '\\' {
+						break // odd number of backslashes
+					}
+					j--
+
 				}
-				j--
 			}
+		} else if c == '\\' {
+			escaped = true
 		}
 	}
 
-	return -1
+	return -1, escaped
 }
 
 // Find end of the data structure, array or object.
@@ -76,7 +87,7 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int {
 	for i < ln {
 		switch data[i] {
 		case '"': // If inside string, skip it
-			se := stringEnd(data[i+1:])
+			se, _ := stringEnd(data[i+1:])
 			if se == -1 {
 				return -1
 			}
@@ -104,13 +115,15 @@ func searchKeys(data []byte, keys ...string) int {
 	ln := len(data)
 	lk := len(keys)
 
+	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
+
 	for i < ln {
 		switch data[i] {
 		case '"':
 			i++
 			keyBegin := i
 
-			strEnd := stringEnd(data[i:])
+			strEnd, keyEscaped := stringEnd(data[i:])
 			if strEnd == -1 {
 				return -1
 			}
@@ -124,12 +137,22 @@ func searchKeys(data []byte, keys ...string) int {
 
 			i += valueOffset
 
-			// if string is a Key, and key level match
-			if data[i] == ':' {
+			// if string is a key, and key level match
+			if data[i] == ':' && keyLevel == level-1 {
 				key := data[keyBegin:keyEnd]
 
-				if keyLevel == level-1 && // If key nesting level match current object nested level
-					equalStr(&key, keys[level-1]) {
+				// for unescape: if there are no escape sequences, this is cheap; if there are, it is a
+				// bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize
+				var keyUnesc []byte
+				if !keyEscaped {
+					keyUnesc = key
+				} else if ku, err := Unescape(key, stackbuf[:]); err != nil {
+					return -1
+				} else {
+					keyUnesc = ku
+				}
+
+				if equalStr(&keyUnesc, keys[level-1]) {
 					keyLevel++
 					// If we found all keys in path
 					if keyLevel == lk {
@@ -206,7 +229,7 @@ func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset
 	// if string value
 	if data[offset] == '"' {
 		dataType = String
-		if idx := stringEnd(data[offset+1:]); idx != -1 {
+		if idx, _ := stringEnd(data[offset+1:]); idx != -1 {
 			endOffset += idx + 1
 		} else {
 			return []byte{}, dataType, offset, MalformedStringError
@@ -371,9 +394,10 @@ func GetString(data []byte, keys ...string) (val string, err error) {
 		return string(v), nil
 	}
 
-	s, err := strconv.Unquote(`"` + string(v) + `"`)
+	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
+	out, err := Unescape(v, stackbuf[:])
 
-	return s, err
+	return string(out), err
 }
 
 // GetFloat returns the value retrieved by `Get`, cast to a float64 if possible.

parser_test.go

@@ -158,6 +158,30 @@ var getTests = []Test{
 		isFound: true,
 		data:    `3`,
 	},
+
+	// Escaped key tests
+	Test{
+		desc:    `key with simple escape`,
+		json:    `{"a\\b":1}`,
+		path:    []string{"a\\b"},
+		isFound: true,
+		data:    `1`,
+	},
+	Test{
+		desc:    `key with Unicode escape`,
+		json:    `{"a\u00B0b":1}`,
+		path:    []string{"a\u00B0b"},
+		isFound: true,
+		data:    `1`,
+	},
+	Test{
+		desc:    `key with complex escape`,
+		json:    `{"a\uD83D\uDE03b":1}`,
+		path:    []string{"a\U0001F603b"},
+		isFound: true,
+		data:    `1`,
+	},
+
 	Test{ // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance
 		desc:    `malformed with trailing whitespace`,
 		json:    `{"a":1 `,
@@ -268,6 +292,7 @@ var getTests = []Test{
 		path:  []string{"a"},
 		isErr: true,
 	},
+
 	Test{ // This test returns not found instead of a parse error, as checking for the malformed JSON would reduce performance
 		desc:  "malformed key (followed by comma followed by colon)",
 		json:  `{"a",:1}`,
@@ -326,19 +351,26 @@ var getFloatTests = []Test{
 
 var getStringTests = []Test{
 	Test{
-		desc:    `Translate unicode symbols`,
+		desc:    `Translate Unicode symbols`,
 		json:    `{"c": "test"}`,
 		path:    []string{"c"},
 		isFound: true,
 		data:    `test`,
 	},
 	Test{
-		desc:    `Translate unicode symbols`,
+		desc:    `Translate Unicode symbols`,
 		json:    `{"c": "15\u00b0C"}`,
 		path:    []string{"c"},
 		isFound: true,
 		data:    `15°C`,
 	},
+	Test{
+		desc:    `Translate supplementary Unicode symbols`,
+		json:    `{"c": "\uD83D\uDE03"}`, // Smiley face (UTF16 surrogate pair)
+		path:    []string{"c"},
+		isFound: true,
+		data:    "\U0001F603", // Smiley face
+	},
 	Test{
 		desc:    `Translate escape symbols`,
 		json:    `{"c": "\\\""}`,