Properly escape filenames in #line directives

In some cases, backslashes in a filenames were already escaped when
emitting a #line directive, but sometimes no escaping happened. In any
case, quotes in a filename should also be escaped.

This commit introduces a helper function `QuoteCppString()` to handle
quoting and escaping strings in a generic way. This new helper function
is used in the code itself, but not in testcases yet.

Note that ctags does not quite properly handle special characters and
escaping in filenames. When parsing line markers, it just ignores
escapes, so any escapes from the #line marker will be present in the
ctags output as well. Before this commit, they would be copied
unmodified into the #line directives generated from the ctags filenames.  Now that these line directives have escaping applied, the filenames read from ctags need to be unescaped, to prevent double escapes. This
unescaping already happened in CollectCTagsFromSketchFiles, but is now
moved to parseTag where it is more appropriate and applies to all users.

Since ctags does not handle escapes in line markers, it cuts the
filename off at the first double quote. If a filename contains a double
quote, the filename as output by ctags will be cut off before it,
leaving the escaping \ at the end. Before this commit, this trailing \
would be output directly into the generated #line marker, which confused
gcc and made the build fail. With this commit, this trailing \ is
escaped itself, making the output syntactically valid, so the build now
works with double quotes in a filename. However, due to this filename
cut off, arduino-builder will no longer generate prototypes for
functions in files that have a double quote in them (since it can no
longer detect them as sketch functions), so double quotes (and probably
tabs too) in filenames are still not completely supported yet.

Signed-off-by: Matthijs Kooijman <matthijs@stdin.nl>

Author: matthijskooijman

src/arduino.cc/builder/collect_ctags_from_sketch_files.go

@@ -32,7 +32,6 @@ package builder
 import (
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
-	"strings"
 )
 
 type CollectCTagsFromSketchFiles struct{}
@@ -43,7 +42,7 @@ func (s *CollectCTagsFromSketchFiles) Run(ctx *types.Context) error {
 	allCtags := ctx.CTagsOfPreprocessedSource
 	ctagsOfSketch := []*types.CTag{}
 	for _, ctag := range allCtags {
-		if utils.SliceContains(sketchFileNames, strings.Replace(ctag.Filename, "\\\\", "\\", -1)) {
+		if utils.SliceContains(sketchFileNames, ctag.Filename) {
 			ctagsOfSketch = append(ctagsOfSketch, ctag)
 		}
 	}

src/arduino.cc/builder/ctags/ctags_parser.go

@@ -229,7 +229,14 @@ func parseTag(row string) *types.CTag {
 	parts := strings.Split(row, "\t")
 
 	tag.FunctionName = parts[0]
-	tag.Filename = parts[1]
+	// This unescapes any backslashes in the filename. These
+	// filenames that ctags outputs originate from the line markers
+	// in the source, as generated by gcc. gcc escapes both
+	// backslashes and double quotes, but ctags ignores any escaping
+	// and just cuts off the filename at the first double quote it
+	// sees. This means any backslashes are still escaped, and need
+	// to be unescape, and any quotes will just break the build.
+	tag.Filename = strings.Replace(parts[1], "\\\\", "\\", -1)
 
 	parts = parts[2:]
 

src/arduino.cc/builder/prototypes_adder.go

@@ -32,6 +32,7 @@ package builder
 import (
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/types"
+	"arduino.cc/builder/utils"
 	"fmt"
 	"strconv"
 	"strings"
@@ -87,7 +88,7 @@ func composePrototypeSection(line int, prototypes []*types.Prototype) string {
 	str := joinPrototypes(prototypes)
 	str += "\n#line "
 	str += strconv.Itoa(line)
-	str += " \"" + prototypes[0].File + "\""
+	str += " " + utils.QuoteCppString(prototypes[0].File)
 	str += "\n"
 
 	return str
@@ -99,7 +100,7 @@ func joinPrototypes(prototypes []*types.Prototype) string {
 		if signatureContainsaDefaultArg(proto) {
 			continue
 		}
-		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" \""+proto.File+"\"")
+		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+utils.QuoteCppString(proto.File))
 		prototypeParts := []string{}
 		if proto.Modifiers != "" {
 			prototypeParts = append(prototypeParts, proto.Modifiers)

src/arduino.cc/builder/sketch_source_merger.go

@@ -32,8 +32,8 @@ package builder
 
 import (
 	"arduino.cc/builder/types"
+	"arduino.cc/builder/utils"
 	"regexp"
-	"strings"
 )
 
 type SketchSourceMerger struct{}
@@ -47,7 +47,7 @@ func (s *SketchSourceMerger) Run(ctx *types.Context) error {
 		includeSection += "#include <Arduino.h>\n"
 		lineOffset++
 	}
-	includeSection += "#line 1 \"" + strings.Replace((&sketch.MainFile).Name, "\\", "\\\\", -1) + "\"\n"
+	includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name) + "\n"
 	lineOffset++
 	ctx.IncludeSection = includeSection
 
@@ -73,7 +73,7 @@ func sketchIncludesArduinoH(sketch *types.SketchFile) bool {
 }
 
 func addSourceWrappedWithLineDirective(sketch *types.SketchFile) string {
-	source := "#line 1 \"" + strings.Replace(sketch.Name, "\\", "\\\\", -1) + "\"\n"
+	source := "#line 1 " + utils.QuoteCppString(sketch.Name) + "\n"
 	source += sketch.Source
 	source += "\n"
 

src/arduino.cc/builder/test/utils_test.go

@@ -87,3 +87,16 @@ func TestMapTrimSpace(t *testing.T) {
 	require.Equal(t, "how are", parts[2])
 	require.Equal(t, "you?", parts[3])
 }
+
+func TestQuoteCppString(t *testing.T) {
+	cases := map[string]string {
+		`foo`: `"foo"`,
+		`foo\bar`: `"foo\\bar"`,
+		`foo "is" quoted and \\bar"" escaped\`: `"foo \"is\" quoted and \\\\bar\"\" escaped\\"`,
+		// ASCII 0x20 - 0x7e, excluding `
+		` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`: `" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`,
+	}
+	for input, expected := range cases {
+		require.Equal(t, expected, utils.QuoteCppString(input))
+	}
+}

src/arduino.cc/builder/utils/utils.go

@@ -407,3 +407,12 @@ func LogIfVerbose(level string, format string, args ...interface{}) types.Comman
 func LogThis(level string, format string, args ...interface{}) types.Command {
 	return &loggerAction{false, level, format, args}
 }
+
+// Returns the given string as a quoted string for use with the C
+// preprocessor. This adds double quotes around it and escapes any
+// double quotes and backslashes in the string.
+func QuoteCppString(str string) string {
+	str = strings.Replace(str, "\\", "\\\\", -1)
+	str = strings.Replace(str, "\"", "\\\"", -1)
+	return "\"" + str + "\""
+}