Merge branch 'pendo-io-faster-safer-keycomp'

Author: buger

Makefile

@@ -1,7 +1,7 @@
 SOURCE = parser.go
 CONTAINER = jsonparser
 SOURCE_PATH = /go/src/github.com/buger/jsonparser
-BENCHMARK = JsonParser
+BENCHMARK = JsonParserSmall
 BENCHTIME = 5s
 TEST = .
 
@@ -14,6 +14,9 @@ race:
 bench:
 	docker run -v `pwd`:$(SOURCE_PATH) -i -t $(CONTAINER) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -benchtime $(BENCHTIME) -v
 
+bench_local:
+	docker run -v `pwd`:$(SOURCE_PATH) -i -t $(CONTAINER) go test $(LDFLAGS) -test.benchmem -bench . $(ARGS) -benchtime $(BENCHTIME) -v
+
 profile:
 	docker run -v `pwd`:$(SOURCE_PATH) -i -t $(CONTAINER) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -memprofile mem.mprof -v
 	docker run -v `pwd`:$(SOURCE_PATH) -i -t $(CONTAINER) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -cpuprofile cpu.out -v

bytes.go

@@ -0,0 +1,28 @@
+package jsonparser
+
+// About 3x faster then strconv.ParseInt because does not check for range error and support only base 10, which is enough for JSON
+func parseInt(bytes []byte) (v int64, ok bool) {
+	if len(bytes) == 0 {
+		return 0, false
+	}
+
+	var neg bool = false
+	if bytes[0] == '-' {
+		neg = true
+		bytes = bytes[1:]
+	}
+
+	for _, c := range bytes {
+		if c >= '0' && c <= '9' {
+			v = (10 * v) + int64(c-'0')
+		} else {
+			return 0, false
+		}
+	}
+
+	if neg {
+		return -v, true
+	} else {
+		return v, true
+	}
+}

bytes_safe.go

@@ -0,0 +1,21 @@
+// +build appengine appenginevm
+
+package jsonparser
+
+import (
+	"strconv"
+)
+
+// See fastbytes_unsafe.go for explanation on why *[]byte is used (signatures must be consistent with those in that file)
+
+func equalStr(b *[]byte, s string) bool {
+	return string(*b) == s
+}
+
+func parseFloat(b *[]byte) (float64, error) {
+	return strconv.ParseFloat(string(*b), 64)
+}
+
+func bytesToString(b *[]byte) string {
+  return string(*b)
+}

bytes_test.go

@@ -0,0 +1,95 @@
+package jsonparser
+
+import (
+	"strconv"
+	"testing"
+	"unsafe"
+)
+
+type ParseIntTest struct {
+	in    string
+	out   int64
+	isErr bool
+}
+
+var parseIntTests = []ParseIntTest{
+	{
+		in:  "0",
+		out: 0,
+	},
+	{
+		in:  "1",
+		out: 1,
+	},
+	{
+		in:  "-1",
+		out: -1,
+	},
+	{
+		in:  "12345",
+		out: 12345,
+	},
+	{
+		in:  "-12345",
+		out: -12345,
+	},
+	{
+		in:  "9223372036854775807",
+		out: 9223372036854775807,
+	},
+	{
+		in:  "-9223372036854775808",
+		out: -9223372036854775808,
+	},
+	{
+		in:  "18446744073709551616", // = 2^64; integer overflow is not detected
+		out: 0,
+	},
+
+	{
+		in:    "",
+		isErr: true,
+	},
+	{
+		in:    "abc",
+		isErr: true,
+	},
+	{
+		in:    "12345x",
+		isErr: true,
+	},
+	{
+		in:    "123e5",
+		isErr: true,
+	},
+	{
+		in:    "9223372036854775807x",
+		isErr: true,
+	},
+}
+
+func TestBytesParseInt(t *testing.T) {
+	for _, test := range parseIntTests {
+		out, ok := parseInt([]byte(test.in))
+		if ok != !test.isErr {
+			t.Errorf("Test '%s' error return did not match expectation (obtained %t, expected %t)", test.in, !ok, test.isErr)
+		} else if ok && out != test.out {
+			t.Errorf("Test '%s' did not return the expected value (obtained %d, expected %d)", test.in, out, test.out)
+		}
+	}
+}
+
+func BenchmarkParseInt(b *testing.B) {
+	bytes := []byte("123")
+	for i := 0; i < b.N; i++ {
+		parseInt(bytes)
+	}
+}
+
+// Alternative implementation using unsafe and delegating to strconv.ParseInt
+func BenchmarkParseIntUnsafeSlower(b *testing.B) {
+	bytes := []byte("123")
+	for i := 0; i < b.N; i++ {
+		strconv.ParseInt(*(*string)(unsafe.Pointer(&bytes)), 10, 64)
+	}
+}

bytes_unsafe.go

@@ -0,0 +1,31 @@
+// +build !appengine,!appenginevm
+
+package jsonparser
+
+import (
+	"strconv"
+	"unsafe"
+)
+
+//
+// The reason for using *[]byte rather than []byte in parameters is an optimization. As of Go 1.6,
+// the compiler cannot perfectly inline the function when using a non-pointer slice. That is,
+// the non-pointer []byte parameter version is slower than if its function body is manually
+// inlined, whereas the pointer []byte version is equally fast to the manually inlined
+// version. Instruction count in assembly taken from "go tool compile" confirms this difference.
+//
+// TODO: Remove hack after Go 1.7 release
+//
+func equalStr(b *[]byte, s string) bool {
+	return *(*string)(unsafe.Pointer(b)) == s
+}
+
+func parseFloat(b *[]byte) (float64, error) {
+	return strconv.ParseFloat(*(*string)(unsafe.Pointer(b)), 64)
+}
+
+// A hack until issue golang/go#2632 is fixed.
+// See: https://github.com/golang/go/issues/2632
+func bytesToString(b *[]byte) string {
+  return *(*string)(unsafe.Pointer(b))
+}

bytes_unsafe_test.go

@@ -0,0 +1,69 @@
+// +build !appengine,!appenginevm
+
+package jsonparser
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+	"unsafe"
+)
+
+var (
+	// short string/[]byte sequences, as the difference between these
+	// three methods is a constant overhead
+	benchmarkString = "0123456789x"
+	benchmarkBytes  = []byte("0123456789y")
+)
+
+func bytesEqualStrSafe(abytes []byte, bstr string) bool {
+	return bstr == string(abytes)
+}
+
+func bytesEqualStrUnsafeSlower(abytes *[]byte, bstr string) bool {
+	aslicehdr := (*reflect.SliceHeader)(unsafe.Pointer(abytes))
+	astrhdr := reflect.StringHeader{Data: aslicehdr.Data, Len: aslicehdr.Len}
+	return *(*string)(unsafe.Pointer(&astrhdr)) == bstr
+}
+
+func TestEqual(t *testing.T) {
+	if !equalStr(&[]byte{}, "") {
+		t.Errorf(`equalStr("", ""): expected true, obtained false`)
+		return
+	}
+
+	longstr := strings.Repeat("a", 1000)
+	for i := 0; i < len(longstr); i++ {
+		s1, s2 := longstr[:i]+"1", longstr[:i]+"2"
+		b1 := []byte(s1)
+
+		if !equalStr(&b1, s1) {
+			t.Errorf(`equalStr("a"*%d + "1", "a"*%d + "1"): expected true, obtained false`, i, i)
+			break
+		}
+		if equalStr(&b1, s2) {
+			t.Errorf(`equalStr("a"*%d + "1", "a"*%d + "2"): expected false, obtained true`, i, i)
+			break
+		}
+	}
+}
+
+func BenchmarkEqualStr(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		equalStr(&benchmarkBytes, benchmarkString)
+	}
+}
+
+// Alternative implementation without using unsafe
+func BenchmarkBytesEqualStrSafe(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		bytesEqualStrSafe(benchmarkBytes, benchmarkString)
+	}
+}
+
+// Alternative implementation using unsafe, but that is slower than the current implementation
+func BenchmarkBytesEqualStrUnsafeSlower(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		bytesEqualStrUnsafeSlower(&benchmarkBytes, benchmarkString)
+	}
+}

parser.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"reflect"
 	"strconv"
-	"unsafe"
 )
 
 // Errors
@@ -127,10 +125,10 @@ func searchKeys(data []byte, keys ...string) int {
 
 			// if string is a Key, and key level match
 			if data[i] == ':' {
-				key := unsafeBytesToString(data[keyBegin:keyEnd])
+				key := data[keyBegin:keyEnd]
 
 				if keyLevel == level-1 && // If key nesting level match current object nested level
-					keys[level-1] == key {
+					equalStr(&key, keys[level-1]) {
 					keyLevel++
 					// If we found all keys in path
 					if keyLevel == lk {
@@ -351,7 +349,7 @@ func GetUnsafeString(data []byte, keys ...string) (val string, err error) {
 		return "", e
 	}
 
-	return unsafeBytesToString(v), nil
+	return bytesToString(&v), nil
 }
 
 // GetString returns the value retrieved by `Get`, cast to a string if possible, trying to properly handle escape and utf8 symbols
@@ -372,7 +370,7 @@ func GetString(data []byte, keys ...string) (val string, err error) {
 		return string(v), nil
 	}
 
-	s, err := strconv.Unquote(`"` + unsafeBytesToString(v) + `"`)
+	s, err := strconv.Unquote(`"` + string(v) + `"`)
 
 	return s, err
 }
@@ -391,7 +389,7 @@ func GetFloat(data []byte, keys ...string) (val float64, err error) {
 		return 0, fmt.Errorf("Value is not a number: %s", string(v))
 	}
 
-	val, err = strconv.ParseFloat(unsafeBytesToString(v), 64)
+	val, err = parseFloat(&v)
 	return
 }
 
@@ -408,8 +406,11 @@ func GetInt(data []byte, keys ...string) (val int64, err error) {
 		return 0, fmt.Errorf("Value is not a number: %s", string(v))
 	}
 
-	val, err = strconv.ParseInt(unsafeBytesToString(v), 10, 64)
-	return
+	if val, ok := parseInt(v); !ok {
+		return 0, MalformedValueError
+	} else {
+		return val, nil
+	}
 }
 
 // GetBoolean returns the value retrieved by `Get`, cast to a bool if possible.
@@ -434,11 +435,3 @@ func GetBoolean(data []byte, keys ...string) (val bool, err error) {
 
 	return
 }
-
-// A hack until issue golang/go#2632 is fixed.
-// See: https://github.com/golang/go/issues/2632
-func unsafeBytesToString(data []byte) string {
-	h := (*reflect.SliceHeader)(unsafe.Pointer(&data))
-	sh := reflect.StringHeader{Data: h.Data, Len: h.Len}
-	return *(*string)(unsafe.Pointer(&sh))
-}