Added recursive library discovery, discovering libraries included by other

Federico Fissore · Federico Fissore · commit ee3687708f9b · 2015-09-18T12:52:42.000+02:00
libraries

Signed-off-by: Federico Fissore &lt;f.fissore@arduino.cc&gt;
diff --git a/src/arduino.cc/builder/add_additional_entries_to_context.go b/src/arduino.cc/builder/add_additional_entries_to_context.go
@@ -75,5 +75,9 @@ func (s *AddAdditionalEntriesToContext) Run(context map[string]interface{}) erro
 		context[constants.CTX_DEBUG_LEVEL] = DEFAULT_DEBUG_LEVEL
 	}
 
+	if !utils.MapHas(context, constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH) {
+		context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH] = DEFAULT_LIBRARY_DISCOVERY_RECURSION_DEPTH
+	}
+
 	return nil
 }
diff --git a/src/arduino.cc/builder/builder.go b/src/arduino.cc/builder/builder.go
@@ -61,6 +61,7 @@ const DEFAULT_DEBUG_LEVEL = 5
 const DEFAULT_WARNINGS_LEVEL = "none"
 const DEFAULT_SOFTWARE = "ARDUINO"
 const DEFAULT_BUILD_CORE = "arduino"
+const DEFAULT_LIBRARY_DISCOVERY_RECURSION_DEPTH = 3
 
 type Builder struct{}
 
diff --git a/src/arduino.cc/builder/constants/constants.go b/src/arduino.cc/builder/constants/constants.go
@@ -65,6 +65,7 @@ const BUILD_PROPERTIES_TOOLS_KEY = "tools"
 const BUILD_PROPERTIES_VID = "vid"
 const COAN = "coan"
 const CTAGS = "ctags"
+const CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH = "libraryDiscoveryRecursionDepth"
 const CTX_ACTUAL_PLATFORM = "actualPlatform"
 const CTX_BUILD_CORE = "buildCore"
 const CTX_BUILD_OPTIONS = "buildOptions"
diff --git a/src/arduino.cc/builder/container_find_includes.go b/src/arduino.cc/builder/container_find_includes.go
@@ -30,24 +30,28 @@
 package builder
 
 import (
+	"arduino.cc/builder/constants"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
 )
 
 type ContainerFindIncludes struct{}
 
 func (s *ContainerFindIncludes) Run(context map[string]interface{}) error {
-	commands := []types.Command{
-		&IncludesFinderWithGCC{},
-		&GCCMinusMOutputParser{},
-		&IncludesToIncludeFolders{},
-	}
+	wheelSpins := context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH].(int)
+	for i := 0; i < wheelSpins+1; i++ {
+		commands := []types.Command{
+			&IncludesFinderWithGCC{},
+			&GCCMinusMOutputParser{},
+			&IncludesToIncludeFolders{},
+		}
 
-	for _, command := range commands {
-		PrintRingNameIfDebug(context, command)
-		err := command.Run(context)
-		if err != nil {
-			return utils.WrapError(err)
+		for _, command := range commands {
+			PrintRingNameIfDebug(context, command)
+			err := command.Run(context)
+			if err != nil {
+				return utils.WrapError(err)
+			}
 		}
 	}
 
diff --git a/src/arduino.cc/builder/gcc_minus_m_output_parser.go b/src/arduino.cc/builder/gcc_minus_m_output_parser.go
@@ -31,6 +31,7 @@ package builder
 
 import (
 	"arduino.cc/builder/constants"
+	"arduino.cc/builder/utils"
 	"strings"
 )
 
@@ -54,7 +55,17 @@ func (s *GCCMinusMOutputParser) Run(context map[string]interface{}) error {
 		}
 	}
 
-	context[constants.CTX_INCLUDES] = includes
+	if !utils.MapHas(context, constants.CTX_INCLUDES) {
+		context[constants.CTX_INCLUDES] = includes
+		return nil
+	}
+
+	previousIncludes := utils.SliceToMapStringBool(context[constants.CTX_INCLUDES].([]string), true)
+	currentIncludes := utils.SliceToMapStringBool(includes, true)
+
+	mergedIncludes := utils.MergeMapsOfStringBool(previousIncludes, currentIncludes)
+
+	context[constants.CTX_INCLUDES] = utils.KeysOfMapOfStringBool(mergedIncludes)
 
 	return nil
 }
diff --git a/src/arduino.cc/builder/hardware/platform.txt b/src/arduino.cc/builder/hardware/platform.txt
@@ -8,7 +8,7 @@ tools.ctags.pattern="{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf -
 tools.avrdude.path={runtime.tools.avrdude.path}
 
 preproc.includes.flags=-w -x c++ -M -MG -MP
-recipe.preproc.includes="{compiler.path}{compiler.cpp.cmd}" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} "{source_file}"
+recipe.preproc.includes="{compiler.path}{compiler.cpp.cmd}" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} "{source_file}"
 
 preproc.macros.flags=-w -x c++ -E -CC
 preproc.macros.compatibility_flags={build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}
diff --git a/src/arduino.cc/builder/includes_finder_with_gcc.go b/src/arduino.cc/builder/includes_finder_with_gcc.go
@@ -36,6 +36,7 @@ import (
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
 	"path/filepath"
+	"strings"
 )
 
 type IncludesFinderWithGCC struct{}
@@ -47,8 +48,16 @@ func (s *IncludesFinderWithGCC) Run(context map[string]interface{}) error {
 	verbose := context[constants.CTX_VERBOSE].(bool)
 	logger := context[constants.CTX_LOGGER].(i18n.Logger)
 
+	includesParams := ""
+	if utils.MapHas(context, constants.CTX_INCLUDE_FOLDERS) {
+		includes := context[constants.CTX_INCLUDE_FOLDERS].([]string)
+		includes = utils.Map(includes, utils.WrapWithHyphenI)
+		includesParams = strings.Join(includes, " ")
+	}
+
 	properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
 	properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp")
+	properties[constants.BUILD_PROPERTIES_INCLUDES] = includesParams
 	builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties)
 
 	output, err := builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_INCLUDES, true, verbose, false, logger)
diff --git a/src/arduino.cc/builder/includes_to_include_folders.go b/src/arduino.cc/builder/includes_to_include_folders.go
@@ -42,14 +42,21 @@ import (
 type IncludesToIncludeFolders struct{}
 
 func (s *IncludesToIncludeFolders) Run(context map[string]interface{}) error {
+	if !utils.MapHas(context, constants.CTX_LIBRARIES) {
+		return nil
+	}
 	includes := context[constants.CTX_INCLUDES].([]string)
 	headerToLibraries := context[constants.CTX_HEADER_TO_LIBRARIES].(map[string][]*types.Library)
 	debugLevel := utils.DebugLevel(context)
 	logger := context[constants.CTX_LOGGER].(i18n.Logger)
 	platform := context[constants.CTX_TARGET_PLATFORM].(*types.Platform)
 	actualPlatform := context[constants.CTX_ACTUAL_PLATFORM].(*types.Platform)
 
-	importedLibraries, err := resolveLibraries(includes, headerToLibraries, []*types.Platform{actualPlatform, platform}, debugLevel, logger)
+	var previousImportedLibraries []*types.Library
+	if utils.MapHas(context, constants.CTX_IMPORTED_LIBRARIES) {
+		previousImportedLibraries = context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)
+	}
+	importedLibraries, err := resolveLibraries(includes, headerToLibraries, previousImportedLibraries, []*types.Platform{actualPlatform, platform}, debugLevel, logger)
 	if err != nil {
 		return utils.WrapError(err)
 	}
@@ -77,17 +84,15 @@ func resolveIncludeFolders(importedLibraries []*types.Library, buildProperties m
 	return includeFolders
 }
 
-func resolveLibraries(includes []string, headerToLibraries map[string][]*types.Library, platforms []*types.Platform, debugLevel int, logger i18n.Logger) ([]*types.Library, error) {
+//FIXME it's also resolving previously resolved libraries
+func resolveLibraries(includes []string, headerToLibraries map[string][]*types.Library, previousImportedLibraries []*types.Library, platforms []*types.Platform, debugLevel int, logger i18n.Logger) ([]*types.Library, error) {
 	markImportedLibrary := make(map[*types.Library]bool)
 	for _, header := range includes {
 		libraries := headerToLibraries[header]
 		if libraries != nil {
 			if len(libraries) == 1 {
 				markImportedLibrary[libraries[0]] = true
 			} else {
-				if debugLevel > 0 {
-					logger.Fprintln(os.Stderr, constants.MSG_LIBRARIES_MULTIPLE_LIBS_FOUND_FOR, header)
-				}
 				var library *types.Library
 				for _, platform := range platforms {
 					if platform != nil && library == nil {
@@ -109,7 +114,8 @@ func resolveLibraries(includes []string, headerToLibraries map[string][]*types.L
 				if library == nil {
 					library = libraries[0]
 				}
-				if debugLevel > 0 {
+				if debugLevel > 0 && !sliceContainsLibrary(previousImportedLibraries, library) {
+					logger.Fprintln(os.Stderr, constants.MSG_LIBRARIES_MULTIPLE_LIBS_FOUND_FOR, header)
 					logger.Fprintln(os.Stderr, constants.MSG_LIBRARIES_USED, library.Folder)
 					for _, notUsedLibrary := range libraries {
 						if library != notUsedLibrary {
@@ -232,3 +238,13 @@ func findLibWithNameContaining(name string, libraries []*types.Library) *types.L
 	}
 	return nil
 }
+
+// thank you golang for s***ing: I can't use/recycle/adapt utils.SliceContains
+func sliceContainsLibrary(slice []*types.Library, target *types.Library) bool {
+	for _, value := range slice {
+		if value == target {
+			return true
+		}
+	}
+	return false
+}
diff --git a/src/arduino.cc/builder/test/add_additional_entries_to_context_test.go b/src/arduino.cc/builder/test/add_additional_entries_to_context_test.go
@@ -51,6 +51,7 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) {
 	require.NotNil(t, context[constants.CTX_WARNINGS_LEVEL])
 	require.NotNil(t, context[constants.CTX_VERBOSE])
 	require.NotNil(t, context[constants.CTX_DEBUG_LEVEL])
+	require.NotNil(t, context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH])
 }
 
 func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {
@@ -69,4 +70,5 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {
 	require.NotNil(t, context[constants.CTX_WARNINGS_LEVEL])
 	require.NotNil(t, context[constants.CTX_VERBOSE])
 	require.NotNil(t, context[constants.CTX_DEBUG_LEVEL])
+	require.NotNil(t, context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH])
 }
diff --git a/src/arduino.cc/builder/test/dependent_libraries/library1/library1.h b/src/arduino.cc/builder/test/dependent_libraries/library1/library1.h
@@ -0,0 +1 @@
+#include <library2.h>
diff --git a/src/arduino.cc/builder/test/dependent_libraries/library2/library2.h b/src/arduino.cc/builder/test/dependent_libraries/library2/library2.h
@@ -0,0 +1 @@
+#include <library3.h>
diff --git a/src/arduino.cc/builder/test/dependent_libraries/library3/library3.h b/src/arduino.cc/builder/test/dependent_libraries/library3/library3.h
@@ -0,0 +1 @@
+#warning included!
diff --git a/src/arduino.cc/builder/test/hardware_loader_test.go b/src/arduino.cc/builder/test/hardware_loader_test.go
@@ -121,7 +121,7 @@ func TestLoadHardwareMixingUserHardwareFolder(t *testing.T) {
 	require.Equal(t, "-w -x c++ -M -MG -MP", avrPlatform.Properties["preproc.includes.flags"])
 	require.Equal(t, "-w -x c++ -E -CC", avrPlatform.Properties["preproc.macros.flags"])
 	require.Equal(t, "{build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}", avrPlatform.Properties["preproc.macros.compatibility_flags"])
-	require.Equal(t, "\"{compiler.path}{compiler.cpp.cmd}\" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} \"{source_file}\"", avrPlatform.Properties[constants.RECIPE_PREPROC_INCLUDES])
+	require.Equal(t, "\"{compiler.path}{compiler.cpp.cmd}\" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} \"{source_file}\"", avrPlatform.Properties[constants.RECIPE_PREPROC_INCLUDES])
 
 	require.NotNil(t, packages["my_avr_platform"])
 	myAVRPlatform := packages["my_avr_platform"].Platforms["avr"]
diff --git a/src/arduino.cc/builder/test/includes_finder_with_gcc_test.go b/src/arduino.cc/builder/test/includes_finder_with_gcc_test.go
@@ -36,6 +36,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
+	"sort"
 	"testing"
 )
 
@@ -61,8 +62,7 @@ func TestIncludesFinderWithGCC(t *testing.T) {
 
 		&builder.ContainerMergeCopySketchFiles{},
 
-		&builder.IncludesFinderWithGCC{},
-		&builder.GCCMinusMOutputParser{},
+		&builder.ContainerFindIncludes{},
 	}
 
 	for _, command := range commands {
@@ -73,6 +73,7 @@ func TestIncludesFinderWithGCC(t *testing.T) {
 	require.NotNil(t, context[constants.CTX_INCLUDES])
 	includes := context[constants.CTX_INCLUDES].([]string)
 	require.Equal(t, 2, len(includes))
+	sort.Strings(includes)
 	require.Equal(t, filepath.Join(buildPath, constants.FOLDER_SKETCH, "empty_1.h"), includes[0])
 	require.Equal(t, filepath.Join(buildPath, constants.FOLDER_SKETCH, "empty_2.h"), includes[1])
 }
@@ -115,3 +116,56 @@ func TestIncludesFinderWithGCCSketchWithConfig(t *testing.T) {
 	require.Equal(t, filepath.Join(buildPath, constants.FOLDER_SKETCH, "includes")+"/de bug.h", includes[1])
 	require.Equal(t, "Bridge.h", includes[2])
 }
+
+func TestIncludesFinderWithGCCSketchWithDependendLibraries(t *testing.T) {
+	DownloadCoresAndToolsAndLibraries(t)
+
+	context := make(map[string]interface{})
+
+	buildPath := SetupBuildPath(t, context)
+	defer os.RemoveAll(buildPath)
+
+	context[constants.CTX_HARDWARE_FOLDERS] = []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}
+	context[constants.CTX_TOOLS_FOLDERS] = []string{"downloaded_tools"}
+	context[constants.CTX_LIBRARIES_FOLDERS] = []string{"dependent_libraries"}
+	context[constants.CTX_FQBN] = "arduino:avr:leonardo"
+	context[constants.CTX_SKETCH_LOCATION] = filepath.Join("sketch_with_dependend_libraries", "sketch.ino")
+	context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
+	context[constants.CTX_VERBOSE] = false
+
+	commands := []types.Command{
+		&builder.SetupHumanLoggerIfMissing{},
+
+		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
+
+		&builder.ContainerMergeCopySketchFiles{},
+
+		&builder.ContainerFindIncludes{},
+	}
+
+	for _, command := range commands {
+		err := command.Run(context)
+		NoError(t, err)
+	}
+
+	require.NotNil(t, context[constants.CTX_INCLUDES])
+	includes := context[constants.CTX_INCLUDES].([]string)
+	require.Equal(t, 6, len(includes))
+
+	sort.Strings(includes)
+	require.Equal(t, Abs(t, filepath.Join("dependent_libraries", "library1", "library1.h")), includes[0])
+	require.Equal(t, Abs(t, filepath.Join("dependent_libraries", "library2", "library2.h")), includes[1])
+	require.Equal(t, Abs(t, filepath.Join("dependent_libraries", "library3", "library3.h")), includes[2])
+	require.Equal(t, "library1.h", includes[3])
+	require.Equal(t, "library2.h", includes[4])
+	require.Equal(t, "library3.h", includes[5])
+
+	require.NotNil(t, context[constants.CTX_IMPORTED_LIBRARIES])
+	importedLibraries := context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)
+	require.Equal(t, 3, len(importedLibraries))
+
+	sort.Sort(ByLibraryName(importedLibraries))
+	require.Equal(t, "library1", importedLibraries[0].Name)
+	require.Equal(t, "library2", importedLibraries[1].Name)
+	require.Equal(t, "library3", importedLibraries[2].Name)
+}
diff --git a/src/arduino.cc/builder/test/libraries/IRremote/IRremote.h b/src/arduino.cc/builder/test/libraries/IRremote/IRremote.h
@@ -1 +1 @@
-#warn IRRemote included!
+#warning IRRemote included!
diff --git a/src/arduino.cc/builder/test/sketch_with_dependend_libraries/sketch.ino b/src/arduino.cc/builder/test/sketch_with_dependend_libraries/sketch.ino
@@ -0,0 +1 @@
+#include <library1.h>
diff --git a/src/arduino.cc/builder/utils/utils.go b/src/arduino.cc/builder/utils/utils.go
@@ -51,6 +51,14 @@ func KeysOfMapOfString(input map[string]string) []string {
 	return keys
 }
 
+func KeysOfMapOfStringBool(input map[string]bool) []string {
+	var keys []string
+	for key, _ := range input {
+		keys = append(keys, key)
+	}
+	return keys
+}
+
 func MergeMapsOfStrings(target map[string]string, sources ...map[string]string) map[string]string {
 	for _, source := range sources {
 		for key, value := range source {
@@ -60,6 +68,15 @@ func MergeMapsOfStrings(target map[string]string, sources ...map[string]string)
 	return target
 }
 
+func MergeMapsOfStringBool(target map[string]bool, sources ...map[string]bool) map[string]bool {
+	for _, source := range sources {
+		for key, value := range source {
+			target[key] = value
+		}
+	}
+	return target
+}
+
 func MergeMapsOfMapsOfStrings(target map[string]map[string]string, sources ...map[string]map[string]string) map[string]map[string]string {
 	for _, source := range sources {
 		for key, value := range source {

Original file line number	Diff line number	Diff line change
`@@ -75,5 +75,9 @@ func (s *AddAdditionalEntriesToContext) Run(context map[string]interface{}) erro`
`75`	`75`	`context[constants.CTX_DEBUG_LEVEL] = DEFAULT_DEBUG_LEVEL`
`76`	`76`	`}`
`77`	`77`
	`78`	`+ if !utils.MapHas(context, constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH) {`
	`79`	`+ context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH] = DEFAULT_LIBRARY_DISCOVERY_RECURSION_DEPTH`
	`80`	`+ }`
	`81`	`+`
`78`	`82`	`return nil`
`79`	`83`	`}`
Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,7 @@ package builder`
`31`	`31`
`32`	`32`	`import (`
`33`	`33`	`"arduino.cc/builder/constants"`
	`34`	`+ "arduino.cc/builder/utils"`
`34`	`35`	`"strings"`
`35`	`36`	`)`
`36`	`37`
`@@ -54,7 +55,17 @@ func (s *GCCMinusMOutputParser) Run(context map[string]interface{}) error {`
`54`	`55`	`}`
`55`	`56`	`}`
`56`	`57`
`57`		`- context[constants.CTX_INCLUDES] = includes`
	`58`	`+ if !utils.MapHas(context, constants.CTX_INCLUDES) {`
	`59`	`+ context[constants.CTX_INCLUDES] = includes`
	`60`	`+ return nil`
	`61`	`+ }`
	`62`	`+`
	`63`	`+ previousIncludes := utils.SliceToMapStringBool(context[constants.CTX_INCLUDES].([]string), true)`
	`64`	`+ currentIncludes := utils.SliceToMapStringBool(includes, true)`
	`65`	`+`
	`66`	`+ mergedIncludes := utils.MergeMapsOfStringBool(previousIncludes, currentIncludes)`
	`67`	`+`
	`68`	`+ context[constants.CTX_INCLUDES] = utils.KeysOfMapOfStringBool(mergedIncludes)`
`58`	`69`
`59`	`70`	`return nil`
`60`	`71`	`}`
Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) {`
`51`	`51`	`require.NotNil(t, context[constants.CTX_WARNINGS_LEVEL])`
`52`	`52`	`require.NotNil(t, context[constants.CTX_VERBOSE])`
`53`	`53`	`require.NotNil(t, context[constants.CTX_DEBUG_LEVEL])`
	`54`	`+ require.NotNil(t, context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH])`
`54`	`55`	`}`
`55`	`56`
`56`	`57`	`func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {`
`@@ -69,4 +70,5 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {`
`69`	`70`	`require.NotNil(t, context[constants.CTX_WARNINGS_LEVEL])`
`70`	`71`	`require.NotNil(t, context[constants.CTX_VERBOSE])`
`71`	`72`	`require.NotNil(t, context[constants.CTX_DEBUG_LEVEL])`
	`73`	`+ require.NotNil(t, context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH])`
`72`	`74`	`}`