Remove arbitrary 64KiB limit on line length

The default bufio.Scanner is limited to 64 * 1024 bytes, throwing a 'token too
long' error if a line exceeds this size. This commit sets the token limit
instead to math.MaxInt, which is the biggest possible slice capacity in Go, and
effectively limits the scan line length only by available memory (fixes #167).

Co-authored-by: Tim Hobbs <timothy.hobbs@servicelayers.com>

Author: bitfield

script.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"os"
 	"os/exec"
@@ -338,7 +339,7 @@ func (p *Pipe) Do(req *http.Request) *Pipe {
 // concurrently and don't do unnecessary reads on the input.
 func (p *Pipe) EachLine(process func(string, *strings.Builder)) *Pipe {
 	return p.Filter(func(r io.Reader, w io.Writer) error {
-		scanner := bufio.NewScanner(r)
+		scanner := newScanner(r)
 		output := strings.Builder{}
 		for scanner.Scan() {
 			process(scanner.Text(), &output)
@@ -413,7 +414,7 @@ func (p *Pipe) ExecForEach(cmdLine string) *Pipe {
 		return p.WithError(err)
 	}
 	return p.Filter(func(r io.Reader, w io.Writer) error {
-		scanner := bufio.NewScanner(r)
+		scanner := newScanner(r)
 		for scanner.Scan() {
 			cmdLine := strings.Builder{}
 			err := tpl.Execute(&cmdLine, scanner.Text())
@@ -502,7 +503,7 @@ func (p *Pipe) FilterLine(filter func(string) string) *Pipe {
 // handling.
 func (p *Pipe) FilterScan(filter func(string, io.Writer)) *Pipe {
 	return p.Filter(func(r io.Reader, w io.Writer) error {
-		scanner := bufio.NewScanner(r)
+		scanner := newScanner(r)
 		for scanner.Scan() {
 			filter(scanner.Text(), w)
 		}
@@ -555,7 +556,7 @@ func (p *Pipe) Freq() *Pipe {
 		count int
 	}
 	return p.Filter(func(r io.Reader, w io.Writer) error {
-		scanner := bufio.NewScanner(r)
+		scanner := newScanner(r)
 		for scanner.Scan() {
 			freq[scanner.Text()]++
 		}
@@ -597,7 +598,7 @@ func (p *Pipe) Get(URL string) *Pipe {
 // space-separated string, which will always end with a newline.
 func (p *Pipe) Join() *Pipe {
 	return p.Filter(func(r io.Reader, w io.Writer) error {
-		scanner := bufio.NewScanner(r)
+		scanner := newScanner(r)
 		var line string
 		first := true
 		for scanner.Scan() {
@@ -659,7 +660,7 @@ func (p *Pipe) Last(n int) *Pipe {
 		return NewPipe()
 	}
 	return p.Filter(func(r io.Reader, w io.Writer) error {
-		scanner := bufio.NewScanner(r)
+		scanner := newScanner(r)
 		input := ring.New(n)
 		for scanner.Scan() {
 			input.Value = scanner.Text()
@@ -938,3 +939,9 @@ func (ra ReadAutoCloser) Read(b []byte) (n int, err error) {
 	}
 	return n, err
 }
+
+func newScanner(r io.Reader) *bufio.Scanner {
+	scanner := bufio.NewScanner(r)
+	scanner.Buffer(make([]byte, 4096), math.MaxInt)
+	return scanner
+}

script_test.go

@@ -270,12 +270,11 @@ func TestDoPerformsSuppliedHTTPRequest(t *testing.T) {
 
 func TestEachLine_FiltersInputThroughSuppliedFunction(t *testing.T) {
 	t.Parallel()
-	p := script.Echo("Hello\nGoodbye")
-	q := p.EachLine(func(line string, out *strings.Builder) {
-		out.WriteString(line + " world\n")
-	})
 	want := "Hello world\nGoodbye world\n"
-	got, err := q.String()
+	got, err := script.Echo("Hello\nGoodbye").
+		EachLine(func(line string, out *strings.Builder) {
+			out.WriteString(line + " world\n")
+		}).String()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -284,6 +283,22 @@ func TestEachLine_FiltersInputThroughSuppliedFunction(t *testing.T) {
 	}
 }
 
+func TestEachLine_HandlesLongLines(t *testing.T) {
+	t.Parallel()
+	var got int
+	_, err := script.Echo(longLine).
+		EachLine(func(line string, out *strings.Builder) {
+			got++
+		}).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := 2
+	if want != got {
+		t.Errorf("want %d lines counted, got %d", want, got)
+	}
+}
+
 func TestEchoProducesSuppliedString(t *testing.T) {
 	t.Parallel()
 	want := "Hello, world."
@@ -328,6 +343,17 @@ func TestExecForEach_ErrorsOnUnbalancedQuotes(t *testing.T) {
 	}
 }
 
+func TestExecForEach_HandlesLongLines(t *testing.T) {
+	t.Parallel()
+	got, err := script.Echo(longLine).ExecForEach(`echo "{{.}}"`).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if longLine != got {
+		t.Error(cmp.Diff(longLine, got))
+	}
+}
+
 func TestFilterByCopyPassesInputThroughUnchanged(t *testing.T) {
 	t.Parallel()
 	p := script.Echo("hello").Filter(func(r io.Reader, w io.Writer) error {
@@ -455,11 +481,29 @@ func TestFilterScan_FiltersInputLineByLine(t *testing.T) {
 	t.Parallel()
 	input := "hello\nworld\ngoodbye"
 	want := "world\n"
-	got, err := script.Echo(input).FilterScan(func(line string, w io.Writer) {
-		if strings.HasPrefix(line, "w") {
-			fmt.Fprintln(w, line)
-		}
-	}).String()
+	got, err := script.Echo(input).
+		FilterScan(func(line string, w io.Writer) {
+			if strings.HasPrefix(line, "w") {
+				fmt.Fprintln(w, line)
+			}
+		}).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want != got {
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
+func TestFilterScan_HandlesLongLines(t *testing.T) {
+	t.Parallel()
+	want := "last line\n"
+	got, err := script.Echo(longLine).
+		FilterScan(func(line string, w io.Writer) {
+			if strings.HasPrefix(line, "last") {
+				fmt.Fprintln(w, line)
+			}
+		}).String()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -520,6 +564,20 @@ func TestFirstHasNoEffectGivenLessThanNInputLines(t *testing.T) {
 	}
 }
 
+func TestFreqHandlesLongLines(t *testing.T) {
+	t.Parallel()
+	got, err := script.Echo(longLine).Freq().Slice()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 results, got %d: %q", len(got), got)
+	}
+	if got[0] != "1 last line" {
+		t.Fatalf("wrong result: %q", got)
+	}
+}
+
 func TestFreqProducesCorrectFrequencyTableForInput(t *testing.T) {
 	t.Parallel()
 	input := strings.Join([]string{
@@ -620,6 +678,18 @@ func TestGetUsesPipeContentsAsRequestBody(t *testing.T) {
 	}
 }
 
+func TestJoinHandlesLongLines(t *testing.T) {
+	t.Parallel()
+	result, err := script.Echo(longLine).Join().String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := len(longLine)
+	if want != len(result) {
+		t.Errorf("want result length %d, got %d", want, len(result))
+	}
+}
+
 func TestJoinJoinsInputLinesIntoSpaceSeparatedString(t *testing.T) {
 	t.Parallel()
 	input := "hello\nfrom\nthe\njoin\ntest"
@@ -726,6 +796,18 @@ func TestLastDropsAllButLastNLinesOfInput(t *testing.T) {
 	}
 }
 
+func TestLastHandlesLongLines(t *testing.T) {
+	t.Parallel()
+	want := "last line\n"
+	got, err := script.Echo(longLine).Last(1).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want != got {
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
 func TestLastHasNoOutputWhenNIs0(t *testing.T) {
 	t.Parallel()
 	input := "a\nb\nc\n"
@@ -2123,3 +2205,8 @@ func ExampleSlice() {
 	// 2
 	// 3
 }
+
+// A string containing a line longer than bufio.MaxScanTokenSize, for testing
+// methods that buffer input. We want to make sure they don't throw
+// "bufio.Scanner: token too long" errors.
+var longLine = strings.Repeat("super long line ", 4096) + "\nlast line\n"