分割ダウンローダーを実装

テストとキャンセル処理は未実装

Author: Shota Sawada

kadai3-2/sawadashota/.gitignore

@@ -0,0 +1,2 @@
+v1.0.11.tar.gz
+vendor/

kadai3-2/sawadashota/Gopkg.lock

@@ -0,0 +1,34 @@
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/sync"
+
+[prune]
+  go-tests = true
+  unused-packages = true

kadai3-2/sawadashota/Gopkg.toml

@@ -0,0 +1,21 @@
+分割ダウンローダー
+===
+
+Download
+---
+
+```bash
+$ go run main.go ${URL}
+```
+
+example
+
+```bash
+$ go run main.go https://github.com/sawadashota/gocmd/archive/v1.0.11.tar.gz
+```
+
+### Options
+
+- `-p`: ダウンロードプロセス数（デフォルトはコア数）
+- `-t`: タイムアウト（デフォルトは10秒）
+- `-o`: 出力先（デフォルトはURLのファイル名）

kadai3-2/sawadashota/README.md

@@ -0,0 +1,85 @@
+package download
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/url"
+	"os"
+	"time"
+)
+
+type Downloader struct {
+	url *url.URL
+	Option
+	Data
+}
+
+type Data struct {
+	filesize uint
+	data     [][]byte
+}
+
+type Option interface {
+	Proc() int
+	Timeout() time.Duration
+	Writer() io.Writer
+	Output() string
+}
+
+// New Downloader
+func New(u *url.URL, opts Option) *Downloader {
+	return &Downloader{
+		url:    u,
+		Option: opts,
+		Data:   Data{},
+	}
+}
+
+// URL getter
+func (d *Downloader) URL() *url.URL {
+	return d.url
+}
+
+// Run download
+func (d *Downloader) Run() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	filesize, err := d.FetchFileSize(d.url.String())
+	if err != nil {
+		return err
+	}
+
+	d.SetFileSize(filesize)
+	d.data = make([][]byte, d.Proc())
+
+	eg := d.get(ctx)
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	body := d.Merge()
+
+	f, err := os.Create(d.Output())
+	defer f.Close()
+
+	f.Write(body)
+
+	return nil
+}
+
+// SetFileSize is filesize setter
+func (d *Data) SetFileSize(size uint) {
+	d.filesize = size
+}
+
+// Merge data slice
+func (d *Data) Merge() []byte {
+	return bytes.Join(d.data, []byte(""))
+}
+
+// String return merged data
+func (d *Data) String() string {
+	return string(d.Merge())
+}

kadai3-2/sawadashota/download/download.go

@@ -0,0 +1,81 @@
+package download
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"golang.org/x/sync/errgroup"
+)
+
+type Range struct {
+	low  int
+	high int
+	proc int
+}
+
+func NewRange(filesize uint, procs, proc int) *Range {
+	split := int(filesize) / procs
+	return &Range{
+		low:  split * (proc - 1),
+		high: split * proc,
+	}
+}
+
+// Low getter
+func (r *Range) Low() int {
+	return r.low
+}
+
+// High getter
+func (r *Range) High() int {
+	return r.high
+}
+
+// FetchFileSize fetch content length and set filesize
+func (d *Data) FetchFileSize(URL string) (uint, error) {
+	resp, err := http.Head(URL)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return uint(resp.ContentLength), nil
+}
+
+// Get contents concurrently
+func (d *Downloader) get(ctx context.Context) *errgroup.Group {
+	eg, ctx := errgroup.WithContext(ctx)
+
+	for i := 0; i < d.Proc(); i++ {
+		i := i
+		r := NewRange(d.filesize, d.Proc(), i+1)
+
+		eg.Go(func() error {
+			req, err := http.NewRequest(http.MethodGet, d.URL().String(), nil)
+			if err != nil {
+				return err
+			}
+
+			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.Low(), r.High()))
+			req = req.WithContext(ctx)
+
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+
+			body, err := ioutil.ReadAll(resp.Body)
+			defer resp.Body.Close()
+			if err != nil {
+				return err
+			}
+
+			d.data[i] = body
+			return nil
+		})
+	}
+
+	return eg
+}

kadai3-2/sawadashota/download/request.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+
+	"fmt"
+	"time"
+
+	"github.com/gopherdojo/dojo2/kadai3-2/sawadashota/download"
+	"github.com/gopherdojo/dojo2/kadai3-2/sawadashota/option"
+)
+
+var writer io.Writer
+
+func init() {
+	writer = os.Stdout
+}
+
+func main() {
+	opts, u, err := option.Parse(writer)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	d := download.New(u, opts)
+	start := time.Now()
+	d.Run()
+	fmt.Fprintf(d.Writer(), "Duration %f seconds\n", time.Since(start).Seconds())
+}

kadai3-2/sawadashota/main.go

@@ -0,0 +1,4 @@
+run:
+	go run main.go https://github.com/sawadashota/gocmd/archive/v1.0.11.tar.gz
+unzip-gz:
+	tar -zxvf v1.0.11.tar.gz

kadai3-2/sawadashota/makefile

@@ -0,0 +1,70 @@
+package option
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/url"
+	"path/filepath"
+	"runtime"
+	"time"
+)
+
+type Option struct {
+	proc    int
+	timeout int
+	output  string
+	writer  io.Writer
+}
+
+// DefaultTimeout is default value for request timeout
+const DefaultTimeout = 10
+
+// Parse CLI options
+func Parse(w io.Writer) (*Option, *url.URL, error) {
+	opts := &Option{
+		writer: w,
+	}
+
+	flag.IntVar(&opts.proc, "p", runtime.NumCPU(), "Parallelism")
+	flag.IntVar(&opts.timeout, "t", DefaultTimeout, "Timeout")
+	flag.StringVar(&opts.output, "o", "", "output")
+
+	flag.Parse()
+
+	if len(flag.Args()) < 1 {
+		return nil, nil, fmt.Errorf("args need url")
+	}
+
+	u, err := url.Parse(flag.Arg(0))
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if opts.output == "" {
+		opts.output = filepath.Base(u.Path)
+	}
+
+	return opts, u, nil
+}
+
+// Proc is proc getter
+func (o *Option) Proc() int {
+	return o.proc
+}
+
+// Timeout is timeout getter
+func (o *Option) Timeout() time.Duration {
+	return time.Duration(o.timeout) * time.Second
+}
+
+// Writer
+func (o *Option) Writer() io.Writer {
+	return o.writer
+}
+
+// Output path
+func (o *Option) Output() string {
+	return o.output
+}