Skip to content

Commit 044817d

Browse files
author
Federico Fissore
committed
gcc -E calls now save output to files. This clutters a bit temp "preproc"
folder, but avoids comunication between go and gcc, which is a performance bottleneck Signed-off-by: Federico Fissore <f.fissore@arduino.cc>
1 parent 3e0793b commit 044817d

24 files changed

+280
-95
lines changed

‎src/arduino.cc/builder/constants/constants.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
6060
const BUILD_PROPERTIES_OBJECT_FILES = "object_files"
6161
const BUILD_PROPERTIES_PATTERN = "pattern"
6262
const BUILD_PROPERTIES_PID = "pid"
63+
const BUILD_PROPERTIES_PREPROCESSED_FILE_PATH = "preprocessed_file_path"
6364
const BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH = "runtime.hardware.path"
6465
const BUILD_PROPERTIES_RUNTIME_OS = "runtime.os"
6566
const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path"
@@ -89,9 +90,10 @@ const CTX_CORE_BUILD_PATH = "coreBuildPath"
8990
const CTX_CTAGS_OF_PREPROC_SOURCE = "ctagsOfPreprocSource"
9091
const CTX_CTAGS_OF_SOURCE = "ctagsOfSource"
9192
const CTX_CTAGS_OUTPUT = "ctagsOutput"
92-
const CTX_CTAGS_TEMP_FILE_NAME = "ctagsTempFileName"
93+
const CTX_CTAGS_TEMP_FILE_PATH = "ctagsTempFilePath"
9394
const CTX_CUSTOM_BUILD_PROPERTIES = "customBuildProperties"
9495
const CTX_DEBUG_LEVEL = "debugLevel"
96+
const CTX_FILE_PATH_TO_READ = "filePathToRead"
9597
const CTX_FOLDERS_WITH_SOURCES_QUEUE = "foldersWithSourcesQueue"
9698
const CTX_FQBN = "fqbn"
9799
const CTX_GCC_MINUS_E_SOURCE = "gccMinusESource"
@@ -252,7 +254,6 @@ const RECIPE_AR_PATTERN = "recipe.ar.pattern"
252254
const RECIPE_C_COMBINE_PATTERN = "recipe.c.combine.pattern"
253255
const RECIPE_C_PATTERN = "recipe.c.o.pattern"
254256
const RECIPE_CPP_PATTERN = "recipe.cpp.o.pattern"
255-
const RECIPE_PREPROC_FINAL = "recipe.preproc.final"
256257
const RECIPE_PREPROC_INCLUDES = "recipe.preproc.includes"
257258
const RECIPE_PREPROC_MACROS = "recipe.preproc.macros"
258259
const RECIPE_S_PATTERN = "recipe.S.o.pattern"

‎src/arduino.cc/builder/container_add_prototypes.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,12 @@ type ContainerAddPrototypes struct{}
3939

4040
func (s *ContainerAddPrototypes) Run(context map[string]interface{}) error {
4141
commands := []types.Command{
42-
&GCCPreprocRunner{},
43-
&CTagsTargetFileSaver{SourceField: constants.CTX_GCC_MINUS_E_SOURCE, Filename: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
42+
&GCCPreprocRunner{TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
43+
&ReadFileAndStoreInContext{TargetField: constants.CTX_GCC_MINUS_E_SOURCE},
44+
&CTagsTargetFileSaver{SourceField: constants.CTX_GCC_MINUS_E_SOURCE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
4445
&CTagsRunner{},
4546
&CTagsParser{CTagsField: constants.CTX_CTAGS_OF_PREPROC_SOURCE},
46-
&CTagsTargetFileSaver{SourceField: constants.CTX_SOURCE, Filename: constants.FILE_CTAGS_TARGET},
47+
&CTagsTargetFileSaver{SourceField: constants.CTX_SOURCE, TargetFileName: constants.FILE_CTAGS_TARGET},
4748
&CTagsRunner{},
4849
&CTagsParser{CTagsField: constants.CTX_CTAGS_OF_SOURCE},
4950
&ComparePrototypesFromSourceAndPreprocSource{},

‎src/arduino.cc/builder/container_find_includes.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ import (
3333
"arduino.cc/builder/constants"
3434
"arduino.cc/builder/types"
3535
"arduino.cc/builder/utils"
36+
"math/rand"
3637
"path/filepath"
38+
"strconv"
3739
)
3840

3941
type ContainerFindIncludes struct{}
@@ -67,10 +69,10 @@ func (s *ContainerFindIncludes) Run(context map[string]interface{}) error {
6769
return utils.WrapError(err)
6870
}
6971

70-
sourceFiles := context[constants.CTX_COLLECTED_SOURCE_FILES_QUEUE].(*types.UniqueStringQueue)
72+
sourceFilePaths := context[constants.CTX_COLLECTED_SOURCE_FILES_QUEUE].(*types.UniqueStringQueue)
7173

72-
for !sourceFiles.Empty() {
73-
err = findIncludesUntilDone(context, sourceFiles.Pop().(string))
74+
for !sourceFilePaths.Empty() {
75+
err = findIncludesUntilDone(context, sourceFilePaths.Pop().(string))
7476
if err != nil {
7577
return utils.WrapError(err)
7678
}
@@ -92,12 +94,13 @@ func runCommand(context map[string]interface{}, command types.Command) error {
9294
return nil
9395
}
9496

95-
func findIncludesUntilDone(context map[string]interface{}, sourceFile string) error {
97+
func findIncludesUntilDone(context map[string]interface{}, sourceFilePath string) error {
98+
targetFileName := filepath.Base(sourceFilePath) + "_" + strconv.Itoa(rand.Int()) + "_preprocessed.cpp"
9699
importedLibraries := context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)
97100
done := false
98101
for !done {
99102
commands := []types.Command{
100-
&GCCPreprocRunnerForDiscoveringIncludes{SourceFile: sourceFile},
103+
&GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourceFilePath, TargetFileName: targetFileName},
101104
&IncludesFinderWithRegExp{ContextField: constants.CTX_GCC_MINUS_E_SOURCE},
102105
&IncludesToIncludeFolders{},
103106
}
@@ -110,7 +113,7 @@ func findIncludesUntilDone(context map[string]interface{}, sourceFile string) er
110113
if len(context[constants.CTX_INCLUDES_JUST_FOUND].([]string)) == 0 {
111114
done = true
112115
} else if len(context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)) == len(importedLibraries) {
113-
err := runCommand(context, &GCCPreprocRunner{})
116+
err := runCommand(context, &GCCPreprocRunner{TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E})
114117
return utils.WrapError(err)
115118
}
116119
importedLibraries = context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)

‎src/arduino.cc/builder/ctags_runner.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,11 @@ type CTagsRunner struct{}
4141

4242
func (s *CTagsRunner) Run(context map[string]interface{}) error {
4343
buildProperties := context[constants.CTX_BUILD_PROPERTIES].(map[string]string)
44-
ctagsTargetFileName := context[constants.CTX_CTAGS_TEMP_FILE_NAME].(string)
44+
ctagsTargetFilePath := context[constants.CTX_CTAGS_TEMP_FILE_PATH].(string)
4545
logger := context[constants.CTX_LOGGER].(i18n.Logger)
4646

4747
properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties, props.SubTree(props.SubTree(buildProperties, constants.BUILD_PROPERTIES_TOOLS_KEY), constants.CTAGS))
48-
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = ctagsTargetFileName
48+
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = ctagsTargetFilePath
4949

5050
pattern := properties[constants.BUILD_PROPERTIES_PATTERN]
5151
if pattern == constants.EMPTY_STRING {

‎src/arduino.cc/builder/ctags_target_file_saver.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ import (
3636
)
3737

3838
type CTagsTargetFileSaver struct {
39-
SourceField string
40-
Filename string
39+
SourceField string
40+
TargetFileName string
4141
}
4242

4343
func (s *CTagsTargetFileSaver) Run(context map[string]interface{}) error {
@@ -49,13 +49,13 @@ func (s *CTagsTargetFileSaver) Run(context map[string]interface{}) error {
4949
return utils.WrapError(err)
5050
}
5151

52-
ctagsTargetFileName := filepath.Join(preprocPath, s.Filename)
53-
err = utils.WriteFile(ctagsTargetFileName, source)
52+
ctagsTargetFilePath := filepath.Join(preprocPath, s.TargetFileName)
53+
err = utils.WriteFile(ctagsTargetFilePath, source)
5454
if err != nil {
5555
return utils.WrapError(err)
5656
}
5757

58-
context[constants.CTX_CTAGS_TEMP_FILE_NAME] = ctagsTargetFileName
58+
context[constants.CTX_CTAGS_TEMP_FILE_PATH] = ctagsTargetFilePath
5959

6060
return nil
6161
}

‎src/arduino.cc/builder/gcc_preproc_runner.go‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,54 +39,71 @@ import (
3939
"strings"
4040
)
4141

42-
type GCCPreprocRunner struct{}
42+
type GCCPreprocRunner struct {
43+
TargetFileName string
44+
}
4345

4446
func (s *GCCPreprocRunner) Run(context map[string]interface{}) error {
4547
sketchBuildPath := context[constants.CTX_SKETCH_BUILD_PATH].(string)
4648
sketch := context[constants.CTX_SKETCH].(*types.Sketch)
47-
properties := prepareGCCPreprocRecipeProperties(context, filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp"))
49+
properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(context, filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp"), s.TargetFileName)
50+
if err != nil {
51+
return utils.WrapError(err)
52+
}
4853

4954
verbose := context[constants.CTX_VERBOSE].(bool)
5055
logger := context[constants.CTX_LOGGER].(i18n.Logger)
51-
output, err := builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_FINAL, true, verbose, false, logger)
56+
_, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger)
5257
if err != nil {
5358
return utils.WrapError(err)
5459
}
5560

56-
context[constants.CTX_GCC_MINUS_E_SOURCE] = string(output)
61+
context[constants.CTX_FILE_PATH_TO_READ] = targetFilePath
5762

5863
return nil
5964
}
6065

6166
type GCCPreprocRunnerForDiscoveringIncludes struct {
62-
SourceFile string
67+
SourceFilePath string
68+
TargetFileName string
6369
}
6470

6571
func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(context map[string]interface{}) error {
66-
properties := prepareGCCPreprocRecipeProperties(context, s.SourceFile)
72+
properties, _, err := prepareGCCPreprocRecipeProperties(context, s.SourceFilePath, s.TargetFileName)
73+
if err != nil {
74+
return utils.WrapError(err)
75+
}
6776

6877
verbose := context[constants.CTX_VERBOSE].(bool)
6978
logger := context[constants.CTX_LOGGER].(i18n.Logger)
70-
output, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger)
79+
stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger)
7180
if err != nil {
7281
return utils.WrapError(err)
7382
}
7483

75-
context[constants.CTX_GCC_MINUS_E_SOURCE] = string(output)
84+
context[constants.CTX_GCC_MINUS_E_SOURCE] = string(stderr)
7685

7786
return nil
7887
}
7988

80-
func prepareGCCPreprocRecipeProperties(context map[string]interface{}, sourceFile string) map[string]string {
89+
func prepareGCCPreprocRecipeProperties(context map[string]interface{}, sourceFilePath string, targetFileName string) (map[string]string, string, error) {
90+
preprocPath := context[constants.CTX_PREPROC_PATH].(string)
91+
err := utils.EnsureFolderExists(preprocPath)
92+
if err != nil {
93+
return nil, "", utils.WrapError(err)
94+
}
95+
targetFilePath := filepath.Join(preprocPath, targetFileName)
96+
8197
buildProperties := utils.GetMapStringStringOrDefault(context, constants.CTX_BUILD_PROPERTIES)
8298
properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
8399

84-
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFile
100+
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath
101+
properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath
85102

86103
includes := context[constants.CTX_INCLUDE_FOLDERS].([]string)
87104
includes = utils.Map(includes, utils.WrapWithHyphenI)
88105
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
89106
builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties)
90107

91-
return properties
108+
return properties, targetFilePath, nil
92109
}

‎src/arduino.cc/builder/hardware/platform.keys.rewrite.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,5 @@ old.11.recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.
3838
new.11.recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.flags} -mmcu={build.mcu} -o "{build.path}/{build.project_name}.elf" {object_files} "{archive_file_path}" "-L{build.path}" -lm
3939

4040
#generic again
41-
old.12.preproc.macros.flags=-w -x c++ -E -CC
42-
new.12.preproc.macros.flags=-w -x c++ -E -CC -dM -include Arduino.h
41+
old.12.recipe.preproc.macros="{compiler.path}{compiler.cpp.cmd}" {compiler.cpp.flags} {preproc.macros.flags} -mmcu={build.mcu} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} "{source_file}"
42+
new.12.recipe.preproc.macros="{compiler.path}{compiler.cpp.cmd}" {compiler.cpp.flags} {preproc.macros.flags} -mmcu={build.mcu} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} "{source_file}" -o "{preprocessed_file_path}"

‎src/arduino.cc/builder/hardware/platform.txt‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,10 @@ tools.ctags.pattern="{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf -
77
# additional entries
88
tools.avrdude.path={runtime.tools.avrdude.path}
99

10-
preproc.includes.flags=-w -x c++ -M -MG -MP -include Arduino.h
10+
preproc.includes.flags=-w -x c++ -M -MG -MP
1111
preproc.includes.compatibility_flags={build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}
1212
recipe.preproc.includes="{compiler.path}{compiler.cpp.cmd}" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {preproc.includes.compatibility_flags} {includes} "{source_file}"
1313

14-
preproc.macros.flags=-w -x c++ -E -CC -dM -include Arduino.h
14+
preproc.macros.flags=-w -x c++ -E -CC
1515
preproc.macros.compatibility_flags={build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}
16-
recipe.preproc.macros="{compiler.path}{compiler.cpp.cmd}" {compiler.cpreprocessor.flags} {compiler.cpp.flags} {preproc.macros.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {preproc.macros.compatibility_flags} {includes} "{source_file}"
17-
18-
preproc.final.flags=-w -x c++ -E -CC -include Arduino.h
19-
preproc.final.compatibility_flags={build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}
20-
recipe.preproc.final="{compiler.path}{compiler.cpp.cmd}" {compiler.cpreprocessor.flags} {compiler.cpp.flags} {preproc.final.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {preproc.final.compatibility_flags} {includes} "{source_file}"
16+
recipe.preproc.macros="{compiler.path}{compiler.cpp.cmd}" {compiler.cpreprocessor.flags} {compiler.cpp.flags} {preproc.macros.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {preproc.macros.compatibility_flags} {includes} "{source_file}" -o "{preprocessed_file_path}"

‎src/arduino.cc/builder/prototypes_adder.go‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,6 @@ func (s *PrototypesAdder) Run(context map[string]interface{}) error {
5757
context[constants.CTX_PROTOTYPE_SECTION] = prototypeSection
5858
source = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
5959

60-
includeSection := composeIncludeArduinoSection()
61-
context[constants.CTX_INCLUDE_SECTION] = includeSection
62-
source = includeSection + source
63-
6460
context[constants.CTX_SOURCE] = source
6561

6662
return nil
@@ -93,13 +89,6 @@ func joinPrototypes(prototypes []*types.Prototype) string {
9389
return strings.Join(prototypesSlice, "\n")
9490
}
9591

96-
func composeIncludeArduinoSection() string {
97-
str := "#include <Arduino.h>\n"
98-
str += "#line 1\n"
99-
100-
return str
101-
}
102-
10392
func firstFunctionOutsideOfSource(firstFunctionLine int, sourceRows []string) bool {
10493
return firstFunctionLine > len(sourceRows)-1
10594
}

‎src/arduino.cc/builder/read_file_and_store_in_context.go‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* This file is part of Arduino Builder.
3+
*
4+
* Arduino Builder is free software; you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation; either version 2 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17+
*
18+
* As a special exception, you may use this file as part of a free software
19+
* library without restriction. Specifically, if other files instantiate
20+
* templates or use macros or inline functions from this file, or you compile
21+
* this file and link it with other files to produce an executable, this
22+
* file does not by itself cause the resulting executable to be covered by
23+
* the GNU General Public License. This exception does not however
24+
* invalidate any other reasons why the executable file might be covered by
25+
* the GNU General Public License.
26+
*
27+
* Copyright 2015 Arduino LLC (http://www.arduino.cc/)
28+
*/
29+
30+
package builder
31+
32+
import (
33+
"arduino.cc/builder/constants"
34+
"arduino.cc/builder/utils"
35+
"io/ioutil"
36+
)
37+
38+
type ReadFileAndStoreInContext struct {
39+
TargetField string
40+
}
41+
42+
func (s *ReadFileAndStoreInContext) Run(context map[string]interface{}) error {
43+
bytes, err := ioutil.ReadFile(context[constants.CTX_FILE_PATH_TO_READ].(string))
44+
if err != nil {
45+
return utils.WrapError(err)
46+
}
47+
48+
context[s.TargetField] = string(bytes)
49+
50+
return nil
51+
}

0 commit comments

Comments
 (0)