Load sketch file contents only when needed

Previously, the full contents of *all* sketch files would be loaded into
memory. This includes all source and header files inside the sketch
directory, even when they will not even be compiled (e.g.
subdirectories other than src). In practice, only the .ino file contents
will actually be used, so these are now read on demand.

Note that when copying the sketch into the build directory, the contents
of all these sketch files *is* used, but that code (`writeIfDifferent()`
in `arduino/builder/sketch.go`) already did not use the preloaded data
but read the file contents when copying.

For small sketches, this does not make much of a difference, but bigger
sketches, especially when they include libraries, core definitions,
tools, examples, documentation, etc. the memory usage can quite explode,
for no good reason.

Author: matthijskooijman

arduino/sketch/sketch.go

@@ -27,25 +27,31 @@ import (
 
 // Item holds the source and the path for a single sketch file
 type Item struct {
-	Path   string
-	Source []byte
+	Path string
 }
 
 // NewItem reads the source code for a sketch item and returns an
 // Item instance
-func NewItem(itemPath string) (*Item, error) {
+func NewItem(itemPath string) *Item {
+	return &Item{itemPath}
+}
+
+func (i *Item) GetSourceBytes() ([]byte, error) {
 	// read the file
-	source, err := ioutil.ReadFile(itemPath)
+	source, err := ioutil.ReadFile(i.Path)
 	if err != nil {
 		return nil, errors.Wrap(err, "error reading source file")
 	}
-
-	return &Item{itemPath, source}, nil
+	return source, nil
 }
 
 // GetSourceStr returns the Source contents in string format
 func (i *Item) GetSourceStr() (string, error) {
-	return string(i.Source), nil
+	source, err := i.GetSourceBytes()
+	if err != nil {
+		return "", err
+	}
+	return string(source), nil
 }
 
 // ItemByPath implements sort.Interface for []Item based on
@@ -73,10 +79,7 @@ func New(sketchFolderPath, mainFilePath, buildPath string, allFilesPaths []strin
 	pathToItem := make(map[string]*Item)
 	for _, p := range allFilesPaths {
 		// create an Item
-		item, err := NewItem(p)
-		if err != nil {
-			return nil, errors.Wrap(err, "error creating the sketch")
-		}
+		item := NewItem(p)
 
 		if p == mainFilePath {
 			// store the main sketch file

arduino/sketch/sketch_test.go

@@ -26,22 +26,26 @@ import (
 
 func TestNewItem(t *testing.T) {
 	sketchItem := filepath.Join("testdata", t.Name()+".ino")
-	item, err := sketch.NewItem(sketchItem)
-	assert.Nil(t, err)
+	item := sketch.NewItem(sketchItem)
 	assert.Equal(t, sketchItem, item.Path)
-	assert.Equal(t, []byte(`#include <testlib.h>`), item.Source)
-	assert.Equal(t, "#include <testlib.h>", item.GetSourceStr())
+	sourceBytes, err := item.GetSourceBytes()
+	assert.Nil(t, err)
+	assert.Equal(t, []byte(`#include <testlib.h>`), sourceBytes)
+	sourceStr, err := item.GetSourceStr()
+	assert.Nil(t, err)
+	assert.Equal(t, "#include <testlib.h>", sourceStr)
 
-	item, err = sketch.NewItem("doesnt/exist")
-	assert.Nil(t, item)
+	item = sketch.NewItem("doesnt/exist")
+	sourceBytes, err = item.GetSourceBytes()
+	assert.Nil(t, sourceBytes)
 	assert.NotNil(t, err)
 }
 
 func TestSort(t *testing.T) {
 	items := []*sketch.Item{
-		&sketch.Item{"foo", nil},
-		&sketch.Item{"baz", nil},
-		&sketch.Item{"bar", nil},
+		&sketch.Item{"foo"},
+		&sketch.Item{"baz"},
+		&sketch.Item{"bar"},
 	}
 
 	sort.Sort(sketch.ItemByPath(items))

legacy/builder/sketch_loader.go

@@ -88,11 +88,7 @@ func collectAllSketchFiles(from *paths.Path) (paths.PathList, error) {
 func makeSketch(sketchLocation *paths.Path, allSketchFilePaths paths.PathList, buildLocation *paths.Path, logger i18n.Logger) (*types.Sketch, error) {
 	sketchFilesMap := make(map[string]types.SketchFile)
 	for _, sketchFilePath := range allSketchFilePaths {
-		source, err := sketchFilePath.ReadFile()
-		if err != nil {
-			return nil, i18n.WrapError(err)
-		}
-		sketchFilesMap[sketchFilePath.String()] = types.SketchFile{Name: sketchFilePath, Source: string(source)}
+		sketchFilesMap[sketchFilePath.String()] = types.SketchFile{Name: sketchFilePath}
 	}
 
 	mainFile := sketchFilesMap[sketchLocation.String()]

legacy/builder/types/types.go

@@ -86,8 +86,7 @@ func (f *SourceFile) DepfilePath(ctx *Context) *paths.Path {
 }
 
 type SketchFile struct {
-	Name   *paths.Path
-	Source string
+	Name *paths.Path
 }
 
 type SketchFileSortByName []SketchFile
@@ -114,20 +113,17 @@ func SketchToLegacy(sketch *sketch.Sketch) *Sketch {
 	s := &Sketch{}
 	s.MainFile = SketchFile{
 		paths.New(sketch.MainFile.Path),
-		string(sketch.MainFile.Source),
 	}
 
 	for _, item := range sketch.OtherSketchFiles {
 		s.OtherSketchFiles = append(s.OtherSketchFiles, SketchFile{
 			paths.New(item.Path),
-			string(item.Source),
 		})
 	}
 
 	for _, item := range sketch.AdditionalFiles {
 		s.AdditionalFiles = append(s.AdditionalFiles, SketchFile{
 			paths.New(item.Path),
-			string(item.Source),
 		})
 	}
 
@@ -137,22 +133,19 @@ func SketchToLegacy(sketch *sketch.Sketch) *Sketch {
 func SketchFromLegacy(s *Sketch) *sketch.Sketch {
 	others := []*sketch.Item{}
 	for _, f := range s.OtherSketchFiles {
-		if i, err := sketch.NewItem(f.Name.String()); err == nil {
-			others = append(others, i)
-		}
+		i := sketch.NewItem(f.Name.String())
+		others = append(others, i)
 	}
 
 	additional := []*sketch.Item{}
 	for _, f := range s.AdditionalFiles {
-		if i, err := sketch.NewItem(f.Name.String()); err == nil {
-			additional = append(additional, i)
-		}
+		i := sketch.NewItem(f.Name.String())
+		additional = append(additional, i)
 	}
 
 	return &sketch.Sketch{
 		MainFile: &sketch.Item{
-			Path:   s.MainFile.Name.String(),
-			Source: []byte(s.MainFile.Source),
+			Path: s.MainFile.Name.String(),
 		},
 		LocationPath:     s.MainFile.Name.Parent().String(),
 		OtherSketchFiles: others,