feat(pieceio): use commp writer

Author: hannahhoward

pieceio/pieceio.go

@@ -4,18 +4,15 @@ import (
 	"context"
 	"io"
 	"os"
-	"sync"
 
 	"github.com/ipfs/go-cid"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipld/go-car"
 	"github.com/ipld/go-ipld-prime"
-	"golang.org/x/xerrors"
 
-	ffi "github.com/filecoin-project/filecoin-ffi"
+	"github.com/filecoin-project/go-commp-utils/writer"
 	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/go-state-types/abi"
 )
 
@@ -79,32 +76,25 @@ func (pio *pieceIO) GeneratePieceReader(payloadCid cid.Cid, selector ipld.Node,
 }
 
 func (pio *pieceIO) GeneratePieceCommitment(rt abi.RegisteredSealProof, payloadCid cid.Cid, selector ipld.Node, storeID *multistore.StoreID, userOnNewCarBlocks ...car.OnNewCarBlockFunc) (cid.Cid, abi.UnpaddedPieceSize, error) {
-	r, pieceSize, err, writeErrChan := pio.GeneratePieceReader(payloadCid, selector, storeID, userOnNewCarBlocks...)
+	bstore, err := pio.bstore(storeID)
 	if err != nil {
 		return cid.Undef, 0, err
 	}
-	commitment, paddedSize, err := GeneratePieceCommitment(rt, r, pieceSize)
-	closeErr := r.Close()
+	preparedCar, err := pio.carIO.PrepareCar(context.Background(), bstore, payloadCid, selector, userOnNewCarBlocks...)
 	if err != nil {
 		return cid.Undef, 0, err
 	}
-	if closeErr != nil {
-		return cid.Undef, 0, closeErr
-	}
-	werr := <-writeErrChan
-	if werr != nil {
-		return cid.Undef, 0, werr
-	}
-	return commitment, paddedSize, nil
-}
 
-func GeneratePieceCommitment(rt abi.RegisteredSealProof, rd io.Reader, pieceSize uint64) (cid.Cid, abi.UnpaddedPieceSize, error) {
-	paddedReader, paddedSize := padreader.New(rd, pieceSize)
-	commitment, err := GeneratePieceCIDFromFile(rt, paddedReader, paddedSize)
+	commpWriter := &writer.Writer{}
+	err = preparedCar.Dump(commpWriter)
 	if err != nil {
 		return cid.Undef, 0, err
 	}
-	return commitment, paddedSize, nil
+	dataCIDSize, err := commpWriter.Sum()
+	if err != nil {
+		return cid.Undef, 0, err
+	}
+	return dataCIDSize.PieceCID, dataCIDSize.PieceSize.Unpadded(), nil
 }
 
 func (pio *pieceIO) ReadPiece(storeID *multistore.StoreID, r io.Reader) (cid.Cid, error) {
@@ -125,61 +115,3 @@ func (pio *pieceIO) bstore(storeID *multistore.StoreID) (blockstore.Blockstore,
 	}
 	return store.Bstore, nil
 }
-
-func ToReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
-	f, ok := r.(*os.File)
-	if ok {
-		return f, func() error { return nil }, nil
-	}
-
-	var w *os.File
-
-	f, w, err := os.Pipe()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var wait sync.Mutex
-	var werr error
-
-	wait.Lock()
-	go func() {
-		defer wait.Unlock()
-
-		var copied int64
-		copied, werr = io.CopyN(w, r, n)
-		if werr != nil {
-			log.Warnf("toReadableFile: copy error: %+v", werr)
-		}
-
-		err := w.Close()
-		if werr == nil && err != nil {
-			werr = err
-			log.Warnf("toReadableFile: close error: %+v", err)
-			return
-		}
-		if copied != n {
-			log.Warnf("copied different amount than expected: %d != %d", copied, n)
-			werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
-		}
-	}()
-
-	return f, func() error {
-		wait.Lock()
-		return werr
-	}, nil
-}
-
-func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader, pieceSize abi.UnpaddedPieceSize) (cid.Cid, error) {
-	f, werr, err := ToReadableFile(piece, int64(pieceSize))
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	pieceCID, err := ffi.GeneratePieceCIDFromFile(proofType, f, pieceSize)
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	return pieceCID, werr()
-}

pieceio/pieceio_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/go-state-types/abi"
 
+	"github.com/filecoin-project/go-commp-utils/ffiwrapper"
 	"github.com/filecoin-project/go-commp-utils/pieceio"
 	"github.com/filecoin-project/go-commp-utils/pieceio/cario"
 	pmocks "github.com/filecoin-project/go-commp-utils/pieceio/mocks"
@@ -132,7 +133,7 @@ func Test_StoreRestoreMemoryBuffer(t *testing.T) {
 
 	paddedReader, secondPaddedSize := padreader.New(r, pieceSize)
 	require.Equal(t, paddedSize, secondPaddedSize)
-	secondCommitment, err := pieceio.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, paddedReader, paddedSize)
+	secondCommitment, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, paddedReader, paddedSize)
 	require.NoError(t, err)
 	require.Equal(t, commitment, secondCommitment)
 	require.NoError(t, r.Close())

writer/writer.go

@@ -1,4 +1,4 @@
-package commp
+package writer
 
 import (
 	"bytes"

writer/writer_test.go

@@ -1,4 +1,4 @@
-package commp
+package writer
 
 import (
 	"bytes"