Merge pull request #1126 from saracen/index-performance-improvements

plumbing: format/index perf, buffered reads, reflection removal

Author: mcuadros

plumbing/format/index/decoder.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"bufio"
 	"bytes"
 	"crypto/sha1"
 	"errors"
@@ -42,14 +43,17 @@ type Decoder struct {
 	r         io.Reader
 	hash      hash.Hash
 	lastEntry *Entry
+
+	extReader *bufio.Reader
 }
 
 // NewDecoder returns a new decoder that reads from r.
 func NewDecoder(r io.Reader) *Decoder {
 	h := sha1.New()
 	return &Decoder{
-		r:    io.TeeReader(r, h),
-		hash: h,
+		r:         io.TeeReader(r, h),
+		hash:      h,
+		extReader: bufio.NewReader(nil),
 	}
 }
 
@@ -184,11 +188,9 @@ func (d *Decoder) doReadEntryNameV4() (string, error) {
 
 func (d *Decoder) doReadEntryName(len uint16) (string, error) {
 	name := make([]byte, len)
-	if err := binary.Read(d.r, &name); err != nil {
-		return "", err
-	}
+	_, err := io.ReadFull(d.r, name[:])
 
-	return string(name), nil
+	return string(name), err
 }
 
 // Index entries are padded out to the next 8 byte alignment
@@ -279,20 +281,21 @@ func (d *Decoder) readExtension(idx *Index, header []byte) error {
 	return nil
 }
 
-func (d *Decoder) getExtensionReader() (io.Reader, error) {
+func (d *Decoder) getExtensionReader() (*bufio.Reader, error) {
 	len, err := binary.ReadUint32(d.r)
 	if err != nil {
 		return nil, err
 	}
 
-	return &io.LimitedReader{R: d.r, N: int64(len)}, nil
+	d.extReader.Reset(&io.LimitedReader{R: d.r, N: int64(len)})
+	return d.extReader, nil
 }
 
 func (d *Decoder) readChecksum(expected []byte, alreadyRead [4]byte) error {
 	var h plumbing.Hash
 	copy(h[:4], alreadyRead[:])
 
-	if err := binary.Read(d.r, h[4:]); err != nil {
+	if _, err := io.ReadFull(d.r, h[4:]); err != nil {
 		return err
 	}
 
@@ -326,7 +329,7 @@ func validateHeader(r io.Reader) (version uint32, err error) {
 }
 
 type treeExtensionDecoder struct {
-	r io.Reader
+	r *bufio.Reader
 }
 
 func (d *treeExtensionDecoder) Decode(t *Tree) error {
@@ -386,16 +389,13 @@ func (d *treeExtensionDecoder) readEntry() (*TreeEntry, error) {
 	}
 
 	e.Trees = i
-
-	if err := binary.Read(d.r, &e.Hash); err != nil {
-		return nil, err
-	}
+	_, err = io.ReadFull(d.r, e.Hash[:])
 
 	return e, nil
 }
 
 type resolveUndoDecoder struct {
-	r io.Reader
+	r *bufio.Reader
 }
 
 func (d *resolveUndoDecoder) Decode(ru *ResolveUndo) error {
@@ -433,7 +433,7 @@ func (d *resolveUndoDecoder) readEntry() (*ResolveUndoEntry, error) {
 
 	for s := range e.Stages {
 		var hash plumbing.Hash
-		if err := binary.Read(d.r, hash[:]); err != nil {
+		if _, err := io.ReadFull(d.r, hash[:]); err != nil {
 			return nil, err
 		}
 
@@ -462,7 +462,7 @@ func (d *resolveUndoDecoder) readStage(e *ResolveUndoEntry, s Stage) error {
 }
 
 type endOfIndexEntryDecoder struct {
-	r io.Reader
+	r *bufio.Reader
 }
 
 func (d *endOfIndexEntryDecoder) Decode(e *EndOfIndexEntry) error {
@@ -472,5 +472,6 @@ func (d *endOfIndexEntryDecoder) Decode(e *EndOfIndexEntry) error {
 		return err
 	}
 
-	return binary.Read(d.r, &e.Hash)
+	_, err = io.ReadFull(d.r, e.Hash[:])
+	return err
 }

storage/filesystem/index.go

@@ -1,6 +1,7 @@
 package filesystem
 
 import (
+	"bufio"
 	"os"
 
 	"gopkg.in/src-d/go-git.v4/plumbing/format/index"
@@ -41,7 +42,7 @@ func (s *IndexStorage) Index() (i *index.Index, err error) {
 
 	defer ioutil.CheckClose(f, &err)
 
-	d := index.NewDecoder(f)
+	d := index.NewDecoder(bufio.NewReader(f))
 	err = d.Decode(idx)
 	return idx, err
 }

utils/binary/read.go

@@ -25,6 +25,10 @@ func Read(r io.Reader, data ...interface{}) error {
 
 // ReadUntil reads from r untin delim is found
 func ReadUntil(r io.Reader, delim byte) ([]byte, error) {
+	if bufr, ok := r.(*bufio.Reader); ok {
+		return ReadUntilFromBufioReader(bufr, delim)
+	}
+
 	var buf [1]byte
 	value := make([]byte, 0, 16)
 	for {
@@ -44,6 +48,17 @@ func ReadUntil(r io.Reader, delim byte) ([]byte, error) {
 	}
 }
 
+// ReadUntilFromBufioReader is like bufio.ReadBytes but drops the delimiter
+// from the result.
+func ReadUntilFromBufioReader(r *bufio.Reader, delim byte) ([]byte, error) {
+	value, err := r.ReadBytes(delim)
+	if err != nil || len(value) == 0 {
+		return nil, err
+	}
+
+	return value[:len(value)-1], nil
+}
+
 // ReadVariableWidthInt reads and returns an int in Git VLQ special format:
 //
 // Ordinary VLQ has some redundancies, example:  the number 358 can be

utils/binary/read_test.go

@@ -1,6 +1,7 @@
 package binary
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/binary"
 	"testing"
@@ -39,6 +40,15 @@ func (s *BinarySuite) TestReadUntil(c *C) {
 	c.Assert(string(b), Equals, "foo")
 }
 
+func (s *BinarySuite) TestReadUntilFromBufioReader(c *C) {
+	buf := bufio.NewReader(bytes.NewBuffer([]byte("foo bar")))
+
+	b, err := ReadUntilFromBufioReader(buf, ' ')
+	c.Assert(err, IsNil)
+	c.Assert(b, HasLen, 3)
+	c.Assert(string(b), Equals, "foo")
+}
+
 func (s *BinarySuite) TestReadVariableWidthInt(c *C) {
 	buf := bytes.NewBuffer([]byte{129, 110})