update to filecoin-ffi with shared types (#127)

* update to filecoin-ffi with shared types

* wire proof type through client

Author: whyrusleeping

pieceio/pieceio.go

@@ -51,15 +51,15 @@ func NewPieceIOWithStore(carIO CarIO, store filestore.FileStore, bs blockstore.B
 	return &pieceIOWithStore{pieceIO{carIO, bs}, store}
 }
 
-func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, abi.UnpaddedPieceSize, error) {
+func (pio *pieceIO) GeneratePieceCommitment(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, abi.UnpaddedPieceSize, error) {
 	preparedCar, err := pio.carIO.PrepareCar(context.Background(), pio.bs, payloadCid, selector)
 	if err != nil {
-		return nil, 0, err
+		return cid.Undef, 0, err
 	}
 	pieceSize := uint64(preparedCar.Size())
 	r, w, err := os.Pipe()
 	if err != nil {
-		return nil, 0, err
+		return cid.Undef, 0, err
 	}
 	var stop sync.WaitGroup
 	stop.Add(1)
@@ -72,25 +72,25 @@ func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.No
 			werr = err
 		}
 	}()
-	commitment, paddedSize, err := GeneratePieceCommitment(r, pieceSize)
+	commitment, paddedSize, err := GeneratePieceCommitment(rt, r, pieceSize)
 	closeErr := r.Close()
 	if err != nil {
-		return nil, 0, err
+		return cid.Undef, 0, err
 	}
 	if closeErr != nil {
-		return nil, 0, closeErr
+		return cid.Undef, 0, closeErr
 	}
 	stop.Wait()
 	if werr != nil {
-		return nil, 0, werr
+		return cid.Undef, 0, werr
 	}
 	return commitment, paddedSize, nil
 }
 
-func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, abi.UnpaddedPieceSize, error) {
+func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, abi.UnpaddedPieceSize, error) {
 	f, err := pio.store.CreateTemp()
 	if err != nil {
-		return nil, "", 0, err
+		return cid.Undef, "", 0, err
 	}
 	cleanup := func() {
 		f.Close()
@@ -99,30 +99,30 @@ func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, s
 	err = pio.carIO.WriteCar(context.Background(), pio.bs, payloadCid, selector, f)
 	if err != nil {
 		cleanup()
-		return nil, "", 0, err
+		return cid.Undef, "", 0, err
 	}
 	pieceSize := uint64(f.Size())
 	_, err = f.Seek(0, io.SeekStart)
 	if err != nil {
 		cleanup()
-		return nil, "", 0, err
+		return cid.Undef, "", 0, err
 	}
-	commitment, paddedSize, err := GeneratePieceCommitment(f, pieceSize)
+	commitment, paddedSize, err := GeneratePieceCommitment(rt, f, pieceSize)
 	if err != nil {
 		cleanup()
-		return nil, "", 0, err
+		return cid.Undef, "", 0, err
 	}
 	_ = f.Close()
 	return commitment, f.Path(), paddedSize, nil
 }
 
-func GeneratePieceCommitment(rd io.Reader, pieceSize uint64) ([]byte, abi.UnpaddedPieceSize, error) {
+func GeneratePieceCommitment(rt abi.RegisteredProof, rd io.Reader, pieceSize uint64) (cid.Cid, abi.UnpaddedPieceSize, error) {
 	paddedReader, paddedSize := padreader.New(rd, pieceSize)
-	commitment, err := sectorbuilder.GeneratePieceCommitment(paddedReader, paddedSize)
+	commitment, err := sectorbuilder.GeneratePieceCIDFromFile(rt, paddedReader, paddedSize)
 	if err != nil {
-		return nil, 0, err
+		return cid.Undef, 0, err
 	}
-	return commitment[:], paddedSize, nil
+	return commitment, paddedSize, nil
 }
 
 func (pio *pieceIO) ReadPiece(r io.Reader) (cid.Cid, error) {

pieceio/pieceio_test.go

@@ -5,21 +5,26 @@ import (
 	"context"
 	"fmt"
 
+	"io"
+	"testing"
+
 	"github.com/filecoin-project/go-fil-markets/filestore"
 	fsmocks "github.com/filecoin-project/go-fil-markets/filestore/mocks"
 	"github.com/filecoin-project/go-fil-markets/pieceio"
 	"github.com/filecoin-project/go-fil-markets/pieceio/cario"
 	pmocks "github.com/filecoin-project/go-fil-markets/pieceio/mocks"
+
 	"github.com/filecoin-project/go-sectorbuilder"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+
+	"github.com/ipfs/go-cid"
 	dag "github.com/ipfs/go-merkledag"
 	dstest "github.com/ipfs/go-merkledag/test"
 	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
 	"github.com/ipld/go-ipld-prime/traversal/selector"
 	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	"io"
-	"testing"
 )
 
 func Test_ThereAndBackAgain(t *testing.T) {
@@ -65,7 +70,7 @@ func Test_ThereAndBackAgain(t *testing.T) {
 			ssb.ExploreIndex(1, ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))))
 	}).Node()
 
-	bytes, tmpPath, _, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node)
+	pcid, tmpPath, _, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 	require.NoError(t, err)
 	tmpFile, err := store.Open(tmpPath)
 	require.NoError(t, err)
@@ -75,9 +80,7 @@ func Test_ThereAndBackAgain(t *testing.T) {
 		deferErr = store.Delete(tmpFile.Path())
 		require.NoError(t, deferErr)
 	}()
-	for _, b := range bytes {
-		require.NotEqual(t, 0, b)
-	}
+	require.NotEqual(t, pcid, cid.Undef)
 	bufSize := int64(16) // small buffer to illustrate the logic
 	buf := make([]byte, bufSize)
 	var readErr error
@@ -155,7 +158,7 @@ func Test_StoreRestoreMemoryBuffer(t *testing.T) {
 			ssb.ExploreIndex(1, ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))))
 	}).Node()
 
-	commitment, tmpPath, paddedSize, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node)
+	commitment, tmpPath, paddedSize, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 	require.NoError(t, err)
 	tmpFile, err := store.Open(tmpPath)
 	require.NoError(t, err)
@@ -169,16 +172,14 @@ func Test_StoreRestoreMemoryBuffer(t *testing.T) {
 	_, err = tmpFile.Seek(0, io.SeekStart)
 	require.NoError(t, err)
 
-	for _, b := range commitment {
-		require.NotEqual(t, 0, b)
-	}
+	require.NotEqual(t, commitment, cid.Undef)
 	buf := make([]byte, paddedSize)
 	_, err = tmpFile.Read(buf)
 	require.NoError(t, err)
 	buffer := bytes.NewBuffer(buf)
-	secondCommitment, err := sectorbuilder.GeneratePieceCommitment(buffer, paddedSize)
+	secondCommitment, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, buffer, paddedSize)
 	require.NoError(t, err)
-	require.Equal(t, commitment, secondCommitment[:])
+	require.Equal(t, commitment, secondCommitment)
 }
 
 func Test_PieceCommitmentEquivalenceMemoryFile(t *testing.T) {
@@ -222,13 +223,13 @@ func Test_PieceCommitmentEquivalenceMemoryFile(t *testing.T) {
 			ssb.ExploreIndex(1, ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()))))
 	}).Node()
 
-	fcommitment, tmpPath, fpaddedSize, ferr := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node)
+	fcommitment, tmpPath, fpaddedSize, ferr := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 	defer func() {
 		deferErr := store.Delete(tmpPath)
 		require.NoError(t, deferErr)
 	}()
 
-	mcommitment, mpaddedSize, merr := pio.GeneratePieceCommitment(nd3.Cid(), node)
+	mcommitment, mpaddedSize, merr := pio.GeneratePieceCommitment(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 	require.Equal(t, fcommitment, mcommitment)
 	require.Equal(t, fpaddedSize, mpaddedSize)
 	require.Equal(t, ferr, merr)
@@ -273,7 +274,7 @@ func Test_Failures(t *testing.T) {
 		fsmock := fsmocks.FileStore{}
 		fsmock.On("CreateTemp").Return(nil, fmt.Errorf("Failed"))
 		pio := pieceio.NewPieceIOWithStore(nil, &fsmock, sourceBs)
-		_, _, _, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node)
+		_, _, _, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 		require.Error(t, err)
 	})
 	t.Run("write CAR fails", func(t *testing.T) {
@@ -285,7 +286,7 @@ func Test_Failures(t *testing.T) {
 		any := mock.Anything
 		ciomock.On("WriteCar", any, any, any, any, any).Return(fmt.Errorf("failed to write car"))
 		pio := pieceio.NewPieceIOWithStore(&ciomock, store, sourceBs)
-		_, _, _, err = pio.GeneratePieceCommitmentToFile(nd3.Cid(), node)
+		_, _, _, err = pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 		require.Error(t, err)
 	})
 	t.Run("prepare CAR fails", func(t *testing.T) {
@@ -294,7 +295,7 @@ func Test_Failures(t *testing.T) {
 		any := mock.Anything
 		ciomock.On("PrepareCar", any, any, any, any).Return(nil, fmt.Errorf("failed to prepare car"))
 		pio := pieceio.NewPieceIO(&ciomock, sourceBs)
-		_, _, err := pio.GeneratePieceCommitment(nd3.Cid(), node)
+		_, _, err := pio.GeneratePieceCommitment(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 		require.Error(t, err)
 	})
 	t.Run("PreparedCard dump operation fails", func(t *testing.T) {
@@ -305,7 +306,7 @@ func Test_Failures(t *testing.T) {
 		preparedCarMock.On("Size").Return(uint64(1000))
 		preparedCarMock.On("Dump", any).Return(fmt.Errorf("failed to write car"))
 		pio := pieceio.NewPieceIO(&ciomock, sourceBs)
-		_, _, err := pio.GeneratePieceCommitment(nd3.Cid(), node)
+		_, _, err := pio.GeneratePieceCommitment(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 		require.Error(t, err)
 	})
 	t.Run("seek fails", func(t *testing.T) {
@@ -332,7 +333,7 @@ func Test_Failures(t *testing.T) {
 		mockfile.On("Seek", mock.Anything, mock.Anything).Return(int64(0), fmt.Errorf("seek failed"))
 
 		pio := pieceio.NewPieceIOWithStore(cio, &fsmock, sourceBs)
-		_, _, _, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node)
+		_, _, _, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node)
 		require.Error(t, err)
 	})
 }

pieceio/types.go

@@ -21,11 +21,11 @@ type ReadStore interface {
 
 // PieceIO converts between payloads and pieces
 type PieceIO interface {
-	GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, abi.UnpaddedPieceSize, error)
+	GeneratePieceCommitment(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, abi.UnpaddedPieceSize, error)
 	ReadPiece(r io.Reader) (cid.Cid, error)
 }
 
 type PieceIOWithStore interface {
 	PieceIO
-	GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, abi.UnpaddedPieceSize, error)
+	GeneratePieceCommitmentToFile(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, abi.UnpaddedPieceSize, error)
 }