Skip to content

Commit 88120ef

Browse files
hannahhowardhannahhoward
committed
feat(commp-utils): initial setup and extraction
1 parent 5c6bf56 commit 88120ef

File tree

15 files changed

+2070
-13
lines changed

15 files changed

+2070
-13
lines changed

‎.circleci/config.yml‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
version: 2.1
2+
orbs:
3+
go: gotest/tools@0.0.9
4+
5+
executors:
6+
golang:
7+
docker:
8+
- image: circleci/golang:1.13
9+
resource_class: medium
10+
11+
commands:
12+
install-deps:
13+
steps:
14+
- go/install-ssh
15+
- go/install: {package: git}
16+
prepare:
17+
parameters:
18+
linux:
19+
default: true
20+
description: is a linux build environment?
21+
type: boolean
22+
darwin:
23+
default: false
24+
description: is a darwin build environment?
25+
type: boolean
26+
steps:
27+
- checkout
28+
- when:
29+
condition: << parameters.linux >>
30+
steps:
31+
- run: sudo apt-get update
32+
- run: sudo apt-get install ocl-icd-opencl-dev
33+
- run: git submodule sync
34+
- run: git submodule update --init
35+
36+
jobs:
37+
mod-tidy-check:
38+
executor: golang
39+
steps:
40+
- install-deps
41+
- prepare
42+
- go/mod-download
43+
- go/mod-tidy-check
44+
45+
build-all:
46+
executor: golang
47+
steps:
48+
- install-deps
49+
- prepare
50+
- go/mod-download
51+
- restore_cache:
52+
name: restore go mod cache
53+
key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
54+
- run:
55+
command: make build
56+
57+
test: &test
58+
description: |
59+
Run tests.
60+
parameters:
61+
executor:
62+
type: executor
63+
default: golang
64+
executor: << parameters.executor >>
65+
steps:
66+
- install-deps
67+
- prepare
68+
- go/mod-download
69+
- restore_cache:
70+
name: restore go mod cache
71+
key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
72+
- run:
73+
name: run tests
74+
command: make test
75+
- save_cache:
76+
name: save go mod cache
77+
key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
78+
paths:
79+
- "~/go/pkg"
80+
- "~/go/src/github.com"
81+
- "~/go/src/golang.org"
82+
83+
build-macos:
84+
description: build with Darwin
85+
macos:
86+
xcode: "10.0.0"
87+
working_directory: ~/go/src/github.com/filecoin-project/go-commp-utils
88+
steps:
89+
- prepare:
90+
linux: false
91+
darwin: true
92+
- run:
93+
name: Install go
94+
command: |
95+
curl -O https://dl.google.com/go/go1.13.4.darwin-amd64.pkg && \
96+
sudo installer -pkg go1.13.4.darwin-amd64.pkg -target /
97+
- run:
98+
name: Install pkg-config
99+
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
100+
- run: go version
101+
- run:
102+
name: Install Rust
103+
command: |
104+
curl https://sh.rustup.rs -sSf | sh -s -- -y
105+
- run:
106+
name: Install jq
107+
command: |
108+
curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
109+
chmod +x /usr/local/bin/jq
110+
- restore_cache:
111+
name: restore go mod and cargo cache
112+
key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/go-commp-utils/go.sum" }}
113+
- install-deps
114+
- go/mod-download
115+
- run:
116+
command: make
117+
- save_cache:
118+
name: save cargo cache
119+
key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/go-commp-utils/go.sum" }}
120+
paths:
121+
- "~/.rustup"
122+
- "~/.cargo"
123+
124+
lint:
125+
description: |
126+
Run golangci-lint.
127+
parameters:
128+
executor:
129+
type: executor
130+
default: golang
131+
golangci-lint-version:
132+
type: string
133+
default: 1.17.1
134+
executor: << parameters.executor >>
135+
steps:
136+
- install-deps
137+
- prepare
138+
- go/mod-download
139+
- run:
140+
command: make
141+
- go/install-golangci-lint:
142+
gobin: $HOME/.local/bin
143+
version: << parameters.golangci-lint-version >>
144+
- run:
145+
name: Lint
146+
command: make lint
147+
148+
workflows:
149+
version: 2.1
150+
ci:
151+
jobs:
152+
- lint
153+
- test
154+
- mod-tidy-check
155+
- build-all
156+
- build-macos

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
**/*.h
2+
**/*.a
3+
**/*.pc
4+
build/.filecoin-ffi-install
5+
build/.update-submodules

‎.gitmodules‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[submodule "extern/filecoin-ffi"]
2+
path = extern/filecoin-ffi
3+
url = git@github.com:filecoin-project/filecoin-ffi.git
4+
branch = master

‎LICENSE-APACHE‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
2+
3+
http://www.apache.org/licenses/LICENSE-2.0
4+
5+
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

‎LICENSE-MIT‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
The MIT License (MIT)
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy
4+
of this software and associated documentation files (the "Software"), to deal
5+
in the Software without restriction, including without limitation the rights
6+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7+
copies of the Software, and to permit persons to whom the Software is
8+
furnished to do so, subject to the following conditions:
9+
10+
The above copyright notice and this permission notice shall be included in
11+
all copies or substantial portions of the Software.
12+
13+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19+
THE SOFTWARE.

‎Makefile‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
SHELL=/usr/bin/env bash
2+
3+
all: build
4+
.PHONY: all
5+
6+
# git submodules that need to be loaded
7+
SUBMODULES:=
8+
9+
# things to clean up, e.g. libfilecoin.a
10+
CLEAN:=
11+
12+
FFI_PATH:=extern/filecoin-ffi/
13+
FFI_DEPS:=.install-filcrypto
14+
FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS))
15+
16+
$(FFI_DEPS): build/.filecoin-ffi-install ;
17+
18+
# dummy file that marks the last time the filecoin-ffi project was built
19+
build/.filecoin-ffi-install: $(FFI_PATH)
20+
$(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%)
21+
@touch $@
22+
23+
SUBMODULES+=$(FFI_PATH)
24+
BUILD_DEPS+=build/.filecoin-ffi-install
25+
CLEAN+=build/.filecoin-ffi-install
26+
27+
$(SUBMODULES): build/.update-submodules ;
28+
29+
# dummy file that marks the last time submodules were updated
30+
build/.update-submodules:
31+
git submodule update --init --recursive
32+
touch $@
33+
34+
CLEAN+=build/.update-submodules
35+
36+
# build and install any upstream dependencies, e.g. filecoin-ffi
37+
deps: $(BUILD_DEPS)
38+
.PHONY: deps
39+
40+
test: $(BUILD_DEPS)
41+
RUST_LOG=info go test -v -timeout 120m ./...
42+
.PHONY: test
43+
44+
lint: $(BUILD_DEPS)
45+
golangci-lint run -v --concurrency 2 --new-from-rev origin/master
46+
.PHONY: lint
47+
48+
build: $(BUILD_DEPS)
49+
go build -v $(GOFLAGS) ./...
50+
.PHONY: build
51+
52+
clean:
53+
rm -rf $(CLEAN)
54+
-$(MAKE) -C $(FFI_PATH) clean
55+
.PHONY: clean

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# go-commp-utils
2+
3+
Utilities for generating and working with CommP for Filecoin pieces
4+
5+
## License
6+
7+
Dual-licensed under [MIT](https://github.com/filecoin-project/go-padreader/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/go-padreader/blob/master/LICENSE-APACHE)

‎extern/filecoin-ffi‎

Submodule filecoin-ffi added at 1d9cb3e

‎ffiwrapper/ffiwrapper.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package ffiwrapper
2+
3+
import (
4+
"io"
5+
"os"
6+
"sync"
7+
8+
ffi "github.com/filecoin-project/filecoin-ffi"
9+
"github.com/filecoin-project/go-state-types/abi"
10+
"github.com/ipfs/go-cid"
11+
logging "github.com/ipfs/go-log/v2"
12+
"golang.org/x/xerrors"
13+
)
14+
15+
var log = logging.Logger("ffi-wrapper")
16+
17+
// ToReadableFile generates an os readable file from an io.Reader (converts via pipe & copy)
18+
func ToReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
19+
f, ok := r.(*os.File)
20+
if ok {
21+
return f, func() error { return nil }, nil
22+
}
23+
24+
var w *os.File
25+
26+
f, w, err := os.Pipe()
27+
if err != nil {
28+
return nil, nil, err
29+
}
30+
31+
var wait sync.Mutex
32+
var werr error
33+
34+
wait.Lock()
35+
go func() {
36+
defer wait.Unlock()
37+
38+
var copied int64
39+
copied, werr = io.CopyN(w, r, n)
40+
if werr != nil {
41+
log.Warnf("toReadableFile: copy error: %+v", werr)
42+
}
43+
44+
err := w.Close()
45+
if werr == nil && err != nil {
46+
werr = err
47+
log.Warnf("toReadableFile: close error: %+v", err)
48+
return
49+
}
50+
if copied != n {
51+
log.Warnf("copied different amount than expected: %d != %d", copied, n)
52+
werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
53+
}
54+
}()
55+
56+
return f, func() error {
57+
wait.Lock()
58+
return werr
59+
}, nil
60+
}
61+
62+
// GeneratePieceCIDFromFile generates an a piece cid from an io.Reader
63+
func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader, pieceSize abi.UnpaddedPieceSize) (cid.Cid, error) {
64+
f, werr, err := ToReadableFile(piece, int64(pieceSize))
65+
if err != nil {
66+
return cid.Undef, err
67+
}
68+
69+
pieceCID, err := ffi.GeneratePieceCIDFromFile(proofType, f, pieceSize)
70+
if err != nil {
71+
return cid.Undef, err
72+
}
73+
74+
return pieceCID, werr()
75+
}

‎go.mod‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
module github.com/filecoin-project/go-commp-utils
2+
3+
go 1.15
4+
5+
require (
6+
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
7+
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
8+
github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
9+
github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc
10+
github.com/filecoin-project/lotus v1.2.0
11+
github.com/ipfs/go-cid v0.0.7
12+
github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
13+
github.com/stretchr/testify v1.6.1
14+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
15+
)
16+
17+
replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

0 commit comments

Comments
 (0)