Add utils.FindFilesInFolder

This is a new helper function that takes care of listing files with
given extensions in a directory, recursively or non-recursively. This
functionality was previously duplicated in a few places. This new helper
function now replaces some duplicate code in SketchLoader and
CollectAllSourcesFilesFromFoldersWithSources. Compilation of code in the
core and libraries still uses two other mechanisms for listing files (in
builder_utils), which will later be converted to use this new helper
function too.

This new helper function has some subtle differences compared to the
previous code:
 - The new code raises an error when the root directory does not exist,
   which the CollectAllSourcesFilesFromFoldersWithSources code did not
   do. To compensate for this, LibraryToSourceFolder will only return
   the "utility" folder when it exists.
 - The new code completely ignores any hidden or source control folders
   or files. Originally, arduino-builder showed a warning when source
   control or hidden folders were present, and then compiled files
   inside of them anyway. Since f5edd23 (Avoid warning about SCCS folders
   in libraries, just ignore them) this warning is no longer shown in
   most cases, but the files themselves are still included in
   compilation. With this commit, they are just completely ignored
   instead (for sketch compilation and include detection, for core and
   library compilation they will are still used). See also #139.

Signed-off-by: Matthijs Kooijman <matthijs@stdin.nl>

Author: matthijskooijman

src/arduino.cc/builder/collect_all_source_files_from_folders_with_sources.go

@@ -30,31 +30,22 @@
 package builder
 
 import (
-	"arduino.cc/builder/gohasissues"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
 )
 
 type CollectAllSourceFilesFromFoldersWithSources struct{}
 
 func (s *CollectAllSourceFilesFromFoldersWithSources) Run(ctx *types.Context) error {
 	foldersWithSources := ctx.FoldersWithSourceFiles
 	sourceFiles := ctx.CollectedSourceFiles
+	extensions := func(ext string) bool { return ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext] }
 
 	filePaths := []string{}
 	for !foldersWithSources.Empty() {
 		sourceFolder := foldersWithSources.Pop().(types.SourceFolder)
-		var err error
-		if sourceFolder.Recurse {
-			err = collectByWalk(&filePaths, sourceFolder.Folder)
-		} else {
-			err = collectByReadDir(&filePaths, sourceFolder.Folder)
-		}
+		err := utils.FindFilesInFolder(&filePaths, sourceFolder.Folder, extensions, sourceFolder.Recurse)
 		if err != nil {
 			return i18n.WrapError(err)
 		}
@@ -66,32 +57,3 @@ func (s *CollectAllSourceFilesFromFoldersWithSources) Run(ctx *types.Context) er
 
 	return nil
 }
-
-func collectByWalk(filePaths *[]string, folder string) error {
-	checkExtensionFunc := func(filePath string) bool {
-		name := filepath.Base(filePath)
-		ext := strings.ToLower(filepath.Ext(filePath))
-		return !strings.HasPrefix(name, ".") && ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext]
-	}
-	walkFunc := utils.CollectAllReadableFiles(filePaths, checkExtensionFunc)
-	err := gohasissues.Walk(folder, walkFunc)
-	return i18n.WrapError(err)
-}
-
-func collectByReadDir(filePaths *[]string, folder string) error {
-	if _, err := os.Stat(folder); err != nil && os.IsNotExist(err) {
-		return nil
-	}
-
-	files, err := ioutil.ReadDir(folder)
-	if err != nil {
-		return i18n.WrapError(err)
-	}
-	for _, file := range files {
-		ext := strings.ToLower(filepath.Ext(file.Name()))
-		if ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext] {
-			*filePaths = append(*filePaths, filepath.Join(folder, file.Name()))
-		}
-	}
-	return nil
-}

src/arduino.cc/builder/sketch_loader.go

@@ -31,7 +31,6 @@ package builder
 
 import (
 	"arduino.cc/builder/constants"
-	"arduino.cc/builder/gohasissues"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
@@ -89,13 +88,8 @@ func (s *SketchLoader) Run(ctx *types.Context) error {
 
 func collectAllSketchFiles(from string) ([]string, error) {
 	filePaths := []string{}
-	checkExtensionFunc := func(filePath string) bool {
-		name := filepath.Base(filePath)
-		ext := strings.ToLower(filepath.Ext(filePath))
-		return !strings.HasPrefix(name, ".") && MAIN_FILE_VALID_EXTENSIONS[ext] || ADDITIONAL_FILE_VALID_EXTENSIONS[ext]
-	}
-	walkFunc := utils.CollectAllReadableFiles(&filePaths, checkExtensionFunc)
-	err := gohasissues.Walk(from, walkFunc)
+	extensions := func(ext string) bool { return MAIN_FILE_VALID_EXTENSIONS[ext] || ADDITIONAL_FILE_VALID_EXTENSIONS[ext] }
+	err := utils.FindFilesInFolder(&filePaths, from, extensions, /* recurse */ true)
 	return filePaths, i18n.WrapError(err)
 }
 

src/arduino.cc/builder/test/collect_all_source_files_from_folders_with_sources_test.go

@@ -106,7 +106,6 @@ func TestCollectAllSourceFilesFromFoldersWithSourcesOfOldLibrary(t *testing.T) {
 	foldersWithSources := &types.UniqueSourceFolderQueue{}
 	foldersWithSources.Push(types.SourceFolder{Folder: Abs(t, filepath.Join("libraries", "ShouldNotRecurseWithOldLibs")), Recurse: false})
 	foldersWithSources.Push(types.SourceFolder{Folder: Abs(t, filepath.Join("libraries", "ShouldNotRecurseWithOldLibs", "utility")), Recurse: false})
-	foldersWithSources.Push(types.SourceFolder{Folder: Abs(t, "non existent folder"), Recurse: false})
 	ctx.FoldersWithSourceFiles = foldersWithSources
 
 	commands := []types.Command{

src/arduino.cc/builder/types/types.go

@@ -34,6 +34,7 @@ import (
 	"arduino.cc/builder/props"
 	"path/filepath"
 	"strconv"
+	"os"
 )
 
 type SketchFile struct {
@@ -201,7 +202,9 @@ func LibraryToSourceFolder(library *Library) []SourceFolder {
 	sourceFolders = append(sourceFolders, SourceFolder{Folder: library.SrcFolder, Recurse: recurse})
 	if library.Layout == LIBRARY_FLAT {
 		utility := filepath.Join(library.SrcFolder, constants.LIBRARY_FOLDER_UTILITY)
-		sourceFolders = append(sourceFolders, SourceFolder{Folder: utility, Recurse: false})
+		if info, err := os.Stat(utility); err == nil && info.IsDir() {
+			sourceFolders = append(sourceFolders, SourceFolder{Folder: utility, Recurse: false})
+		}
 	}
 	return sourceFolders
 }

src/arduino.cc/builder/utils/utils.go

@@ -304,30 +304,48 @@ func FilterOutFoldersByNames(folders []os.FileInfo, names ...string) []os.FileIn
 	return filtered
 }
 
-type CheckFilePathFunc func(filePath string) bool
+type CheckExtensionFunc func(ext string) bool
 
-func CollectAllReadableFiles(collector *[]string, test CheckFilePathFunc) filepath.WalkFunc {
-	walkFunc := func(currentPath string, info os.FileInfo, err error) error {
+func FindFilesInFolder(files *[]string, folder string, extensions CheckExtensionFunc, recurse bool) error {
+	walkFunc := func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if info.IsDir() {
+		// Skip source control and hidden files and directories
+		if IsSCCSOrHiddenFile(info) {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
 			return nil
 		}
-		if !test(currentPath) {
+
+		// Skip directories unless recurse is on, or this is the
+		// root directory
+		if info.IsDir() {
+			if recurse || path == folder {
+				return nil
+			} else {
+				return filepath.SkipDir
+			}
+		}
+
+		// Check (lowercased) extension against list of extensions
+		if extensions != nil && !extensions(strings.ToLower(filepath.Ext(path))) {
 			return nil
 		}
-		currentFile, err := os.Open(currentPath)
+
+		// See if the file is readable by opening it
+		currentFile, err := os.Open(path)
 		if err != nil {
 			return nil
 		}
 		currentFile.Close()
 
-		*collector = append(*collector, currentPath)
+		*files = append(*files, path)
 		return nil
 	}
-	return walkFunc
+	return gohasissues.Walk(folder, walkFunc)
 }
 
 func AppendIfNotPresent(target []string, elements ...string) []string {