Only pass sketch sources to ctags

Previously, the .ino files from the sketch were concatenated,
preprocessed and then the entire preprocessing result was passed through
ctags. Since ctags isn't a perfect C/C++ parser, this would cause it to
sometimes break on some complex constructs from library or system header
files, preventing the entire sketch from compiling.

This commit filters the result of preprocessing to only include lines
that originate from the original sketch files and filter out any lines
from included system or library header files. This filtering happens
based on the line markers that the gcc preprocessor emits.

Because of this, the CollectCTagsFromSketchFiles pass can be removed.
This pass used to apply the same filtering, but on the tags generated by
ctags. Since now only sketch lines are fed into ctags, the resulting
ctags will obviously all originate from sketch files, so no need to
further filter them.

With this change, all sketches from the test suite still compile as
expected. However, there is a fair chance that with this change, ctags
will fail to parse some sketches. In general, to parse a C or C++ file,
you need to keep track of the type environment and symbol table, since
identifiers can either be variables or types, depending on the context.
When lines from included files are cut off, this information is not
available, and ctags will have to guess (though it might be doing that
anyway, not sure if it normally does completely proper parsing).

Signed-off-by: Matthijs Kooijman <matthijs@stdin.nl>

Author: matthijskooijman

src/arduino.cc/builder/container_add_prototypes.go

@@ -42,10 +42,10 @@ func (s *ContainerAddPrototypes) Run(ctx *types.Context) error {
 	commands := []types.Command{
 		&GCCPreprocRunner{TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
 		&ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE},
+		&FilterSketchSource{Source: &ctx.SourceGccMinusE},
 		&CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
 		&ctags.CTagsRunner{},
 		&ctags.CTagsParser{},
-		&CollectCTagsFromSketchFiles{},
 		&ctags.CTagsToPrototypes{},
 		&PrototypesAdder{},
 		&SketchSaver{},

src/arduino.cc/builder/ctags/ctags_to_prototypes.go

@@ -37,7 +37,7 @@ import (
 type CTagsToPrototypes struct{}
 
 func (s *CTagsToPrototypes) Run(ctx *types.Context) error {
-	tags := ctx.CTagsCollected
+	tags := ctx.CTagsOfPreprocessedSource
 
 	lineWhereToInsertPrototypes := findLineWhereToInsertPrototypes(tags)
 	if lineWhereToInsertPrototypes != -1 {

src/arduino.cc/builder/collect_ctags_from_sketch_files.go renamed to src/arduino.cc/builder/filter_sketch_source.go

@@ -32,30 +32,68 @@ package builder
 import (
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
+	"strconv"
+	"strings"
 )
 
-type CollectCTagsFromSketchFiles struct{}
+type FilterSketchSource struct {
+	Source         *string
+}
 
-func (s *CollectCTagsFromSketchFiles) Run(ctx *types.Context) error {
-	sketchFileNames := collectSketchFileNamesFrom(ctx.Sketch)
+func (s *FilterSketchSource) Run(ctx *types.Context) error {
+	lines := strings.Split(*s.Source, "\n")
 
-	allCtags := ctx.CTagsOfPreprocessedSource
-	ctagsOfSketch := []*types.CTag{}
-	for _, ctag := range allCtags {
-		if utils.SliceContains(sketchFileNames, ctag.Filename) {
-			ctagsOfSketch = append(ctagsOfSketch, ctag)
-		}
+	fileNames := []string{ctx.Sketch.MainFile.Name}
+	for _, file := range ctx.Sketch.OtherSketchFiles {
+		fileNames = append(fileNames, file.Name)
 	}
 
-	ctx.CTagsCollected = ctagsOfSketch
+	inSketch := false
+	filtered := ""
+
+	for _, line := range lines {
+		filename := parseLineMarker(line)
+		if filename != "" {
+			inSketch = utils.SliceContains(fileNames, filename)
+		}
+
+		if inSketch {
+			filtered += line + "\n"
+		}
+	}
 
+	*s.Source = filtered
 	return nil
 }
 
-func collectSketchFileNamesFrom(sketch *types.Sketch) []string {
-	fileNames := []string{sketch.MainFile.Name}
-	for _, file := range sketch.OtherSketchFiles {
-		fileNames = append(fileNames, file.Name)
+// Parses the given line as a gcc line marker and returns the contained
+// filename.
+func parseLineMarker(line string) string {
+	// A line marker contains the line number and filename and looks like:
+	// # 123 /path/to/file.cpp
+	// It can be followed by zero or more flag number that indicate the
+	// preprocessor state and can be ignored.
+	// For exact details on this format, see:
+	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
+
+	split := strings.SplitN(line, " ", 3)
+	if len(split) < 2 || split[0] != "#" {
+		return ""
+	}
+
+	_, err := strconv.Atoi(split[1])
+	if err != nil {
+		return ""
 	}
-	return fileNames
+
+	// If we get here, we found a # followed by a line number, so
+	// assume this is a line marker and see if the rest of the line
+	// starts with a string containing the filename
+	str, rest, ok := utils.ParseCppString(split[2])
+
+	if ok && (rest == "" || rest[0] == ' ') {
+		return str
+	}
+	return ""
 }
+

src/arduino.cc/builder/test/ctags_to_prototypes_test.go

@@ -38,13 +38,6 @@ import (
 	"testing"
 )
 
-type CollectCtagsFromPreprocSource struct{}
-
-func (*CollectCtagsFromPreprocSource) Run(ctx *types.Context) error {
-	ctx.CTagsCollected = ctx.CTagsOfPreprocessedSource
-	return nil
-}
-
 func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) {
 	ctx := &types.Context{}
 
@@ -55,7 +48,6 @@ func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -87,7 +79,6 @@ func TestCTagsToPrototypesShouldListTemplates(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -117,7 +108,6 @@ func TestCTagsToPrototypesShouldListTemplates2(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -148,7 +138,6 @@ func TestCTagsToPrototypesShouldDealWithClasses(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -174,7 +163,6 @@ func TestCTagsToPrototypesShouldDealWithStructs(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -204,7 +192,6 @@ func TestCTagsToPrototypesShouldDealWithMacros(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -236,7 +223,6 @@ func TestCTagsToPrototypesShouldDealFunctionWithDifferentSignatures(t *testing.T
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -264,7 +250,6 @@ func TestCTagsToPrototypesClassMembersAreFilteredOut(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -293,7 +278,6 @@ func TestCTagsToPrototypesStructWithFunctions(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -322,7 +306,6 @@ func TestCTagsToPrototypesDefaultArguments(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -352,7 +335,6 @@ func TestCTagsToPrototypesNamespace(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -381,7 +363,6 @@ func TestCTagsToPrototypesStatic(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -412,7 +393,6 @@ func TestCTagsToPrototypesFunctionPointer(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -442,7 +422,6 @@ func TestCTagsToPrototypesFunctionPointers(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 

src/arduino.cc/builder/types/context.go

@@ -66,7 +66,6 @@ type Context struct {
 	CTagsOutput                 string
 	CTagsTargetFile             string
 	CTagsOfPreprocessedSource   []*CTag
-	CTagsCollected              []*CTag
 	IncludeSection              string
 	LineOffset                  int
 	PrototypesSection           string