Merge pull request #1006 from mcuadros/transactional-storage

storage: transactional, new storage with transactional capabilities

Author: mcuadros

plumbing/storer/object.go

@@ -222,7 +222,7 @@ type MultiEncodedObjectIter struct {
 }
 
 // NewMultiEncodedObjectIter returns an object iterator for the given slice of
-// objects.
+// EncodedObjectIters.
 func NewMultiEncodedObjectIter(iters []EncodedObjectIter) EncodedObjectIter {
 	return &MultiEncodedObjectIter{iters: iters}
 }

plumbing/storer/reference.go

@@ -131,25 +131,87 @@ func (iter *ReferenceSliceIter) Next() (*plumbing.Reference, error) {
 // an error happens or the end of the iter is reached. If ErrStop is sent
 // the iteration is stop but no error is returned. The iterator is closed.
 func (iter *ReferenceSliceIter) ForEach(cb func(*plumbing.Reference) error) error {
+	return forEachReferenceIter(iter, cb)
+}
+
+type bareReferenceIterator interface {
+	Next() (*plumbing.Reference, error)
+	Close()
+}
+
+func forEachReferenceIter(iter bareReferenceIterator, cb func(*plumbing.Reference) error) error {
 	defer iter.Close()
-	for _, r := range iter.series {
-		if err := cb(r); err != nil {
+	for {
+		obj, err := iter.Next()
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return err
+		}
+
+		if err := cb(obj); err != nil {
 			if err == ErrStop {
 				return nil
 			}
 
 			return err
 		}
 	}
-
-	return nil
 }
 
 // Close releases any resources used by the iterator.
 func (iter *ReferenceSliceIter) Close() {
 	iter.pos = len(iter.series)
 }
 
+// MultiReferenceIter implements ReferenceIter. It iterates over several
+// ReferenceIter,
+//
+// The MultiReferenceIter must be closed with a call to Close() when it is no
+// longer needed.
+type MultiReferenceIter struct {
+	iters []ReferenceIter
+}
+
+// NewMultiReferenceIter returns an reference iterator for the given slice of
+// EncodedObjectIters.
+func NewMultiReferenceIter(iters []ReferenceIter) ReferenceIter {
+	return &MultiReferenceIter{iters: iters}
+}
+
+// Next returns the next reference from the iterator, if one iterator reach
+// io.EOF is removed and the next one is used.
+func (iter *MultiReferenceIter) Next() (*plumbing.Reference, error) {
+	if len(iter.iters) == 0 {
+		return nil, io.EOF
+	}
+
+	obj, err := iter.iters[0].Next()
+	if err == io.EOF {
+		iter.iters[0].Close()
+		iter.iters = iter.iters[1:]
+		return iter.Next()
+	}
+
+	return obj, err
+}
+
+// ForEach call the cb function for each reference contained on this iter until
+// an error happens or the end of the iter is reached. If ErrStop is sent
+// the iteration is stop but no error is returned. The iterator is closed.
+func (iter *MultiReferenceIter) ForEach(cb func(*plumbing.Reference) error) error {
+	return forEachReferenceIter(iter, cb)
+}
+
+// Close releases any resources used by the iterator.
+func (iter *MultiReferenceIter) Close() {
+	for _, i := range iter.iters {
+		i.Close()
+	}
+}
+
 // ResolveReference resolves a SymbolicReference to a HashReference.
 func ResolveReference(s ReferenceStorer, n plumbing.ReferenceName) (*plumbing.Reference, error) {
 	r, err := s.Reference(n)

plumbing/storer/reference_test.go

@@ -172,3 +172,26 @@ func (s *ReferenceSuite) TestReferenceFilteredIterForEachStop(c *C) {
 
 	c.Assert(count, Equals, 1)
 }
+
+func (s *ReferenceSuite) TestMultiReferenceIterForEach(c *C) {
+	i := NewMultiReferenceIter(
+		[]ReferenceIter{
+			NewReferenceSliceIter([]*plumbing.Reference{
+				plumbing.NewReferenceFromStrings("foo", "foo"),
+			}),
+			NewReferenceSliceIter([]*plumbing.Reference{
+				plumbing.NewReferenceFromStrings("bar", "bar"),
+			}),
+		},
+	)
+
+	var result []string
+	err := i.ForEach(func(r *plumbing.Reference) error {
+		result = append(result, r.Name().String())
+		return nil
+	})
+
+	c.Assert(err, IsNil)
+	c.Assert(result, HasLen, 2)
+	c.Assert(result, DeepEquals, []string{"foo", "bar"})
+}

storage/filesystem/dotgit/dotgit.go

@@ -14,6 +14,7 @@ import (
 
 	"gopkg.in/src-d/go-billy.v4/osfs"
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/storage"
 	"gopkg.in/src-d/go-git.v4/utils/ioutil"
 
 	"gopkg.in/src-d/go-billy.v4"
@@ -596,7 +597,7 @@ func (d *DotGit) checkReferenceAndTruncate(f billy.File, old *plumbing.Reference
 		return err
 	}
 	if ref.Hash() != old.Hash() {
-		return fmt.Errorf("reference has changed concurrently")
+		return storage.ErrReferenceHasChanged
 	}
 	_, err = f.Seek(0, io.SeekStart)
 	if err != nil {

storage/memory/storage.go

@@ -13,7 +13,6 @@ import (
 )
 
 var ErrUnsupportedObjectType = fmt.Errorf("unsupported object type")
-var ErrRefHasChanged = fmt.Errorf("reference has changed concurrently")
 
 // Storage is an implementation of git.Storer that stores data on memory, being
 // ephemeral. The use of this storage should be done in controlled envoriments,
@@ -258,7 +257,7 @@ func (r ReferenceStorage) CheckAndSetReference(ref, old *plumbing.Reference) err
 	if old != nil {
 		tmp := r[ref.Name()]
 		if tmp != nil && tmp.Hash() != old.Hash() {
-			return ErrRefHasChanged
+			return storage.ErrReferenceHasChanged
 		}
 	}
 	r[ref.Name()] = ref

storage/storer.go

@@ -1,10 +1,14 @@
 package storage
 
 import (
+	"errors"
+
 	"gopkg.in/src-d/go-git.v4/config"
 	"gopkg.in/src-d/go-git.v4/plumbing/storer"
 )
 
+var ErrReferenceHasChanged = errors.New("reference has changed concurrently")
+
 // Storer is a generic storage of objects, references and any information
 // related to a particular repository. The package gopkg.in/src-d/go-git.v4/storage
 // contains two implementation a filesystem base implementation (such as `.git`)

storage/test/storage_suite.go

@@ -280,6 +280,57 @@ func (s *BaseStorageSuite) TestSetReferenceAndGetReference(c *C) {
 	c.Assert(e.Hash().String(), Equals, "bc9968d75e48de59f0870ffb71f5e160bbbdcf52")
 }
 
+func (s *BaseStorageSuite) TestCheckAndSetReference(c *C) {
+	err := s.Storer.SetReference(
+		plumbing.NewReferenceFromStrings("foo", "482e0eada5de4039e6f216b45b3c9b683b83bfa"),
+	)
+	c.Assert(err, IsNil)
+
+	err = s.Storer.CheckAndSetReference(
+		plumbing.NewReferenceFromStrings("foo", "bc9968d75e48de59f0870ffb71f5e160bbbdcf52"),
+		plumbing.NewReferenceFromStrings("foo", "482e0eada5de4039e6f216b45b3c9b683b83bfa"),
+	)
+	c.Assert(err, IsNil)
+
+	e, err := s.Storer.Reference(plumbing.ReferenceName("foo"))
+	c.Assert(err, IsNil)
+	c.Assert(e.Hash().String(), Equals, "bc9968d75e48de59f0870ffb71f5e160bbbdcf52")
+}
+
+func (s *BaseStorageSuite) TestCheckAndSetReferenceNil(c *C) {
+	err := s.Storer.SetReference(
+		plumbing.NewReferenceFromStrings("foo", "482e0eada5de4039e6f216b45b3c9b683b83bfa"),
+	)
+	c.Assert(err, IsNil)
+
+	err = s.Storer.CheckAndSetReference(
+		plumbing.NewReferenceFromStrings("foo", "bc9968d75e48de59f0870ffb71f5e160bbbdcf52"),
+		nil,
+	)
+	c.Assert(err, IsNil)
+
+	e, err := s.Storer.Reference(plumbing.ReferenceName("foo"))
+	c.Assert(err, IsNil)
+	c.Assert(e.Hash().String(), Equals, "bc9968d75e48de59f0870ffb71f5e160bbbdcf52")
+}
+
+func (s *BaseStorageSuite) TestCheckAndSetReferenceError(c *C) {
+	err := s.Storer.SetReference(
+		plumbing.NewReferenceFromStrings("foo", "c3f4688a08fd86f1bf8e055724c84b7a40a09733"),
+	)
+	c.Assert(err, IsNil)
+
+	err = s.Storer.CheckAndSetReference(
+		plumbing.NewReferenceFromStrings("foo", "bc9968d75e48de59f0870ffb71f5e160bbbdcf52"),
+		plumbing.NewReferenceFromStrings("foo", "482e0eada5de4039e6f216b45b3c9b683b83bfa"),
+	)
+	c.Assert(err, Equals, storage.ErrReferenceHasChanged)
+
+	e, err := s.Storer.Reference(plumbing.ReferenceName("foo"))
+	c.Assert(err, IsNil)
+	c.Assert(e.Hash().String(), Equals, "c3f4688a08fd86f1bf8e055724c84b7a40a09733")
+}
+
 func (s *BaseStorageSuite) TestRemoveReference(c *C) {
 	err := s.Storer.SetReference(
 		plumbing.NewReferenceFromStrings("foo", "bc9968d75e48de59f0870ffb71f5e160bbbdcf52"),

storage/transactional/config.go

@@ -0,0 +1,50 @@
+package transactional
+
+import "gopkg.in/src-d/go-git.v4/config"
+
+// ConfigStorage implements the storer.ConfigStorage for the transactional package.
+type ConfigStorage struct {
+	config.ConfigStorer
+	temporal config.ConfigStorer
+
+	set bool
+}
+
+// NewConfigStorage returns a new ConfigStorer based on a base storer and a
+// temporal storer.
+func NewConfigStorage(s, temporal config.ConfigStorer) *ConfigStorage {
+	return &ConfigStorage{ConfigStorer: s, temporal: temporal}
+}
+
+// SetConfig honors the storer.ConfigStorer interface.
+func (c *ConfigStorage) SetConfig(cfg *config.Config) error {
+	if err := c.temporal.SetConfig(cfg); err != nil {
+		return err
+	}
+
+	c.set = true
+	return nil
+}
+
+// Config honors the storer.ConfigStorer interface.
+func (c *ConfigStorage) Config() (*config.Config, error) {
+	if !c.set {
+		return c.ConfigStorer.Config()
+	}
+
+	return c.temporal.Config()
+}
+
+// Commit it copies the config from the temporal storage into the base storage.
+func (c *ConfigStorage) Commit() error {
+	if !c.set {
+		return nil
+	}
+
+	cfg, err := c.temporal.Config()
+	if err != nil {
+		return err
+	}
+
+	return c.ConfigStorer.SetConfig(cfg)
+}

storage/transactional/config_test.go

@@ -0,0 +1,82 @@
+package transactional
+
+import (
+	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/config"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+var _ = Suite(&ConfigSuite{})
+
+type ConfigSuite struct{}
+
+func (s *ConfigSuite) TestSetConfigBase(c *C) {
+	cfg := config.NewConfig()
+	cfg.Core.Worktree = "foo"
+
+	base := memory.NewStorage()
+	err := base.SetConfig(cfg)
+	c.Assert(err, IsNil)
+
+	temporal := memory.NewStorage()
+	cs := NewConfigStorage(base, temporal)
+
+	cfg, err = cs.Config()
+	c.Assert(err, IsNil)
+	c.Assert(cfg.Core.Worktree, Equals, "foo")
+}
+
+func (s *ConfigSuite) TestSetConfigTemporal(c *C) {
+	cfg := config.NewConfig()
+	cfg.Core.Worktree = "foo"
+
+	base := memory.NewStorage()
+	err := base.SetConfig(cfg)
+	c.Assert(err, IsNil)
+
+	temporal := memory.NewStorage()
+
+	cfg = config.NewConfig()
+	cfg.Core.Worktree = "bar"
+
+	cs := NewConfigStorage(base, temporal)
+	err = cs.SetConfig(cfg)
+	c.Assert(err, IsNil)
+
+	baseCfg, err := base.Config()
+	c.Assert(err, IsNil)
+	c.Assert(baseCfg.Core.Worktree, Equals, "foo")
+
+	temporalCfg, err := temporal.Config()
+	c.Assert(err, IsNil)
+	c.Assert(temporalCfg.Core.Worktree, Equals, "bar")
+
+	cfg, err = cs.Config()
+	c.Assert(err, IsNil)
+	c.Assert(temporalCfg.Core.Worktree, Equals, "bar")
+}
+
+func (s *ConfigSuite) TestCommit(c *C) {
+	cfg := config.NewConfig()
+	cfg.Core.Worktree = "foo"
+
+	base := memory.NewStorage()
+	err := base.SetConfig(cfg)
+	c.Assert(err, IsNil)
+
+	temporal := memory.NewStorage()
+
+	cfg = config.NewConfig()
+	cfg.Core.Worktree = "bar"
+
+	cs := NewConfigStorage(base, temporal)
+	err = cs.SetConfig(cfg)
+	c.Assert(err, IsNil)
+
+	err = cs.Commit()
+	c.Assert(err, IsNil)
+
+	baseCfg, err := base.Config()
+	c.Assert(err, IsNil)
+	c.Assert(baseCfg.Core.Worktree, Equals, "bar")
+}

storage/transactional/doc.go

@@ -0,0 +1,7 @@
+// Package transactional is a transactional implementation of git.Storer, it
+// demux the write and read operation of two separate storers, allowing to merge
+// content calling Storage.Commit.
+//
+// The API and functionality of this package are considered EXPERIMENTAL and is
+// not considered stable nor production ready.
+package transactional