Do not limit includes to the src sketch subdirectory

Commit 405a24a (Limit recursive sketch compilation to the `src`
directory) modified the sketch loading to only recursively load files
from the `src` subdirectory of a sketch, in order to only compile source
files in that subdirectory. As a side effect, this also prevents source
and header files in other subdirectories from being copied into the
build directory, and hence from being included. This side effect was not
entirely unanticipated, but its implications were underestimated.

This commit undoes these side effects: source and header files are again
loaded (and thus copied) recursively in all subdirectories of a sketch.
Compilation is still limited to the src subdirectory (there already was
code to achieve this). This commit also modifies include detection to be
limited to the src subdirectory (which was previously implicit, since it
can only look at files that were copied into the build directory).

With this commit, the the meaning of "loading" a file becomes a bit
fuzzy in the code, but this is intended to be fixed in a future
refactor.

A test will be added in the next commit.

This fixes arduino/Arduino#5186.

Signed-off-by: Cristian Maglie <c.maglie@arduino.cc>
Signed-off-by: Matthijs Kooijman <matthijs@stdin.nl>

Author: cmaglie

src/arduino.cc/builder/container_find_includes.go

@@ -30,11 +30,13 @@
 package builder
 
 import (
+	"os"
+	"path/filepath"
+
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
-	"path/filepath"
 )
 
 type ContainerFindIncludes struct{}
@@ -53,7 +55,11 @@ func (s *ContainerFindIncludes) Run(ctx *types.Context) error {
 	}
 
 	foldersWithSources := ctx.FoldersWithSourceFiles
-	foldersWithSources.Push(types.SourceFolder{Folder: ctx.SketchBuildPath, Recurse: true})
+	foldersWithSources.Push(types.SourceFolder{Folder: ctx.SketchBuildPath, Recurse: false})
+	srcSubfolderPath := filepath.Join(ctx.SketchBuildPath, constants.SKETCH_FOLDER_SRC)
+	if info, err := os.Stat(srcSubfolderPath); err == nil && info.IsDir() {
+		foldersWithSources.Push(types.SourceFolder{Folder: srcSubfolderPath, Recurse: true})
+	}
 	if len(ctx.ImportedLibraries) > 0 {
 		for _, library := range ctx.ImportedLibraries {
 			sourceFolders := types.LibraryToSourceFolder(library)

src/arduino.cc/builder/sketch_loader.go

@@ -92,18 +92,11 @@ func collectAllSketchFiles(from string) ([]string, error) {
 	// Source files in the root are compiled, non-recursively. This
 	// is the only place where .ino files can be present.
 	rootExtensions := func(ext string) bool { return MAIN_FILE_VALID_EXTENSIONS[ext] || ADDITIONAL_FILE_VALID_EXTENSIONS[ext] }
-	err := utils.FindFilesInFolder(&filePaths, from, rootExtensions, false /* recurse */)
+	err := utils.FindFilesInFolder(&filePaths, from, rootExtensions, true /* recurse */)
 	if err != nil {
 		return nil, i18n.WrapError(err)
 	}
 
-	// The "src/" subdirectory of a sketch is compiled recursively
-	// (but .ino files are *not* compiled)
-	srcPath := filepath.Join(from, constants.SKETCH_FOLDER_SRC)
-	if info, err := os.Stat(srcPath); err == nil && info.IsDir() {
-		srcExtensions := func(ext string) bool { return ADDITIONAL_FILE_VALID_EXTENSIONS[ext] }
-		err = utils.FindFilesInFolder(&filePaths, srcPath, srcExtensions, true /* recurse */)
-	}
 	return filePaths, i18n.WrapError(err)
 }
 

src/arduino.cc/builder/test/sketch_loader_test.go

@@ -30,11 +30,12 @@
 package test
 
 import (
+	"path/filepath"
+	"testing"
+
 	"arduino.cc/builder"
 	"arduino.cc/builder/types"
 	"github.com/stretchr/testify/require"
-	"path/filepath"
-	"testing"
 )
 
 func TestLoadSketchWithFolder(t *testing.T) {
@@ -122,9 +123,11 @@ func TestLoadSketchFromFolder(t *testing.T) {
 
 	require.Equal(t, 0, len(sketch.OtherSketchFiles))
 
-	require.Equal(t, 2, len(sketch.AdditionalFiles))
-	require.Contains(t, sketch.AdditionalFiles[0].Name, "other.cpp")
-	require.Contains(t, sketch.AdditionalFiles[1].Name, "other.h")
+	require.Equal(t, 4, len(sketch.AdditionalFiles))
+	require.Contains(t, sketch.AdditionalFiles[0].Name, "sketch_with_subfolders/src/subfolder/other.cpp")
+	require.Contains(t, sketch.AdditionalFiles[1].Name, "sketch_with_subfolders/src/subfolder/other.h")
+	require.Contains(t, sketch.AdditionalFiles[2].Name, "sketch_with_subfolders/subfolder/dont_load_me.cpp")
+	require.Contains(t, sketch.AdditionalFiles[3].Name, "sketch_with_subfolders/subfolder/other.h")
 }
 
 func TestLoadSketchWithBackup(t *testing.T) {