Merge pull request #10193 from bridadan/arm_tc_with_fallback

Fallback to ARMC5 when ARMC6 is not configured

Author: Nir Sonnenschein

targets/targets.json

@@ -2594,7 +2594,7 @@
     },
     "MTB_MXCHIP_EMW3166": {
         "inherits": ["FAMILY_STM32"],
-        "supported_toolchains": ["ARM", "GCC_ARM", "IAR"],
+        "supported_toolchains": ["ARMC6", "GCC_ARM", "IAR"],
         "core": "Cortex-M4F",
         "extra_labels_add": [
             "STM32F4",
@@ -2627,7 +2627,7 @@
     },
     "USI_WM_BN_BM_22": {
         "inherits": ["FAMILY_STM32"],
-        "supported_toolchains": ["ARM", "GCC_ARM", "IAR"],
+        "supported_toolchains": ["ARMC6", "GCC_ARM", "IAR"],
         "components_add": ["SPIF", "FLASHIAP"],
         "core": "Cortex-M4F",
         "extra_labels_add": [
@@ -5309,7 +5309,7 @@
     "RZ_A1XX": {
         "inherits": ["Target"],
         "core": "Cortex-A9",
-        "supported_toolchains": ["ARM", "GCC_ARM", "IAR"],
+        "supported_toolchains": ["ARMC6", "GCC_ARM", "IAR"],
         "extra_labels": ["RENESAS", "RZ_A1XX"],
         "device_has": [
             "SLEEP",

tools/build.py

@@ -29,24 +29,24 @@
 sys.path.insert(0, ROOT)
 
 
-from tools.toolchains import TOOLCHAINS, TOOLCHAIN_CLASSES, TOOLCHAIN_PATHS
-from tools.toolchains import mbedToolchain
-from tools.targets import TARGET_NAMES, TARGET_MAP, Target
+from tools.toolchains import TOOLCHAINS
+from tools.targets import TARGET_NAMES, Target
 from tools.options import get_default_options_parser
 from tools.options import extract_profile
 from tools.options import extract_mcus
 from tools.build_api import build_library, build_mbed_libs, build_lib
 from tools.build_api import mcu_toolchain_matrix
 from tools.build_api import print_build_results
-from tools.build_api import get_toolchain_name
-from tools.settings import CPPCHECK_CMD, CPPCHECK_MSG_FORMAT
-from tools.settings import CPPCHECK_CMD, CPPCHECK_MSG_FORMAT, CLI_COLOR_MAP
+from tools.build_api import target_supports_toolchain
+from tools.build_api import find_valid_toolchain
 from tools.notifier.term import TerminalNotifier
 from tools.utils import argparse_filestring_type, args_error, argparse_many
-from tools.utils import argparse_filestring_type, argparse_dir_not_parent
+from tools.utils import argparse_dir_not_parent
+from tools.utils import NoValidToolchainException
+from tools.utils import print_end_warnings
 from tools.paths import is_relative_to_root
 
-if __name__ == '__main__':
+def main():
     start = time()
 
     # Parse Options
@@ -169,40 +169,37 @@
     failures = []
     successes = []
     skipped = []
+    end_warnings = []
 
-    toolchain_names = set()
     for toolchain in toolchains:
         for target_name in targets:
             target = Target.get_target(target_name)
-            toolchain_names.add(get_toolchain_name(target, toolchain))
 
-    for toolchain_name in toolchain_names:
-        if not TOOLCHAIN_CLASSES[toolchain_name].check_executable():
-            search_path = TOOLCHAIN_PATHS[toolchain_name] or "No path set"
-            args_error(parser, "Could not find executable for %s.\n"
-                               "Currently set search path: %s"
-                       % (toolchain_name, search_path))
+            try:
+                toolchain_name, internal_tc_name, end_warnings = find_valid_toolchain(
+                    target, toolchain
+                )
+            except NoValidToolchainException as e:
+                print_end_warnings(e.end_warnings)
+                args_error(parser, str(e))
 
-    for toolchain in toolchains:
-        for target in targets:
-            tt_id = "%s::%s" % (toolchain, target)
-            if toolchain not in TARGET_MAP[target].supported_toolchains:
+            tt_id = "%s::%s" % (internal_tc_name, target_name)
+            if not target_supports_toolchain(target, toolchain):
                 # Log this later
                 print("%s skipped: toolchain not supported" % tt_id)
                 skipped.append(tt_id)
             else:
                 try:
                     notifier = TerminalNotifier(options.verbose, options.silent)
-                    mcu = TARGET_MAP[target]
-                    profile = extract_profile(parser, options, toolchain)
+                    profile = extract_profile(parser, options, internal_tc_name)
 
-                    if mcu.is_PSA_secure_target and \
+                    if target.is_PSA_secure_target and \
                             not is_relative_to_root(options.source_dir):
                         options.source_dir = ROOT
 
                     if options.source_dir:
                         lib_build_res = build_library(
-                            options.source_dir, options.build_dir, mcu, toolchain,
+                            options.source_dir, options.build_dir, target, toolchain_name,
                             jobs=options.jobs,
                             clean=options.clean,
                             archive=(not options.no_archive),
@@ -214,7 +211,7 @@
                         )
                     else:
                         lib_build_res = build_mbed_libs(
-                            mcu, toolchain,
+                            target, toolchain_name,
                             jobs=options.jobs,
                             clean=options.clean,
                             macros=options.macros,
@@ -225,7 +222,7 @@
 
                     for lib_id in libraries:
                         build_lib(
-                            lib_id, mcu, toolchain,
+                            lib_id, target, toolchain_name,
                             clean=options.clean,
                             macros=options.macros,
                             jobs=options.jobs,
@@ -236,10 +233,15 @@
                         successes.append(tt_id)
                     else:
                         skipped.append(tt_id)
+                except KeyboardInterrupt as e:
+                    print("\n[CTRL+c] exit")
+                    print_end_warnings(end_warnings)
+                    sys.exit(0)
                 except Exception as e:
                     if options.verbose:
                         import traceback
                         traceback.print_exc(file=sys.stdout)
+                        print_end_warnings(end_warnings)
                         sys.exit(1)
                     failures.append(tt_id)
                     print(e)
@@ -254,5 +256,10 @@
         if report:
             print(print_build_results(report, report_name))
 
+    print_end_warnings(end_warnings)
     if failures:
         sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tools/build_api.py

@@ -34,7 +34,7 @@
 from .arm_pack_manager import Cache
 from .utils import (mkdir, run_cmd, run_cmd_ext, NotSupportedException,
                     ToolException, InvalidReleaseTargetException,
-                    copy_when_different)
+                    copy_when_different, NoValidToolchainException)
 from .paths import (MBED_CMSIS_PATH, MBED_TARGETS_PATH, MBED_LIBRARIES,
                     MBED_HEADER, MBED_DRIVERS, MBED_PLATFORM, MBED_HAL,
                     MBED_CONFIG_FILE, MBED_LIBRARIES_DRIVERS,
@@ -44,7 +44,8 @@
 from .notifier.mock import MockNotifier
 from .targets import TARGET_NAMES, TARGET_MAP, CORE_ARCH, Target
 from .libraries import Library
-from .toolchains import TOOLCHAIN_CLASSES
+from .toolchains import TOOLCHAIN_CLASSES, TOOLCHAIN_PATHS
+from .toolchains.arm import ARMC5_MIGRATION_WARNING
 from .config import Config
 
 RELEASE_VERSIONS = ['2', '5']
@@ -120,18 +121,76 @@ def add_result_to_report(report, result):
     result_wrap = {0: result}
     report[target][toolchain][id_name].append(result_wrap)
 
+def get_valid_toolchain_names(target, toolchain_name):
+    """Return the list of toolchains with which a build should be attempted. This
+    list usually contains one element, however there may be multiple entries if
+    a toolchain is expected to fallback to different versions depending on the
+    environment configuration. If an invalid supported_toolchain configuration
+    is detected, an Exception will be raised.
+
+    Positional arguments:
+    target - Target object (not the string name) of the device we are building for
+    toolchain_name - the string that identifies the build toolchain as supplied by
+    the front-end scripts
+    """
+    if int(target.build_tools_metadata["version"]) > 0:
+        all_arm_toolchain_names = ["ARMC6", "ARMC5", "ARM"]
+        arm_st = set(target.supported_toolchains).intersection(
+            set(all_arm_toolchain_names)
+        )
+        if len(arm_st) > 1:
+            raise Exception(
+                "Targets may only specify one of the following in "
+                "supported_toolchains: {}\n"
+                "The following toolchains were present: {}".format(
+                    ", ".join(all_arm_toolchain_names),
+                    ", ".join(arm_st),
+                )
+            )
+        if toolchain_name == "ARM":
+            # The order matters here
+            all_arm_toolchain_names = ["ARMC6", "ARMC5"]
+            if "ARM" in target.supported_toolchains:
+                return all_arm_toolchain_names
+
+            result_list = []
+            for tc_name in all_arm_toolchain_names:
+                if tc_name in target.supported_toolchains:
+                    result_list.append(tc_name)
+            return result_list
+
+    return [toolchain_name]
+
 def get_toolchain_name(target, toolchain_name):
+    """Get the internal toolchain name given the toolchain_name provided by
+    the front-end scripts (usually by the -t/--toolchain argument) and the target
+
+    Positional arguments:
+    target - Target object (not the string name) of the device we are building for
+    toolchain_name - the string that identifies the build toolchain as supplied by
+    the front-end scripts
+
+    Overview of what the current return values should be for the "ARM" family of
+    toolchains (since the behavior is fairly complex). Top row header represents
+    the argument "toolchain_name", Left column header represents the attribute
+    "supported_toolchains" of the "target" argument.
+
+    | supported_toolchains/toolchain_name |+| ARMC5 | ARMC6 | ARM    |
+    ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+    | ARMC5                               |+| ARM*  | ARMC6 | ARM    |
+    | ARMC6                               |+| ARM*  | ARMC6 | ARMC6* |
+    | ARM                                 |+| ARM*  | ARMC6 | ARMC6* |
+
+    * Denotes that the input "toolchain_name" changes in the return value
+    """
     if int(target.build_tools_metadata["version"]) > 0:
-        if toolchain_name == "ARM" or toolchain_name == "ARMC6" :
-            if("ARM" in target.supported_toolchains or "ARMC6" in target.supported_toolchains):
+        if toolchain_name == "ARMC5":
+            return "ARM"
+        elif toolchain_name == "ARM":
+            if set(target.supported_toolchains).intersection(set(["ARMC6", "ARM"])):
                 return "ARMC6"
-            elif ("ARMC5" in target.supported_toolchains):
-                if toolchain_name == "ARM":
-                    return "ARM" #note that returning ARM here means, use ARMC5 toolchain
-                else:
-                    return "ARMC6" #ARMC6 explicitly specified by user, try ARMC6 anyway although the target doesnt explicitly specify ARMC6, as ARMC6 is our default ARM toolchain
         elif toolchain_name == "uARM":
-            if ("ARMC5" in target.supported_toolchains):
+            if "ARMC5" in target.supported_toolchains:
                 return "uARM" #use ARM_MICRO to use AC5+microlib
             else:
                 return "ARMC6" #use AC6+microlib
@@ -144,6 +203,51 @@ def get_toolchain_name(target, toolchain_name):
 
     return toolchain_name
 
+def find_valid_toolchain(target, toolchain):
+    """Given a target and toolchain, get the names for the appropriate
+    toolchain to use. The environment is also checked to see if the corresponding
+    compiler is configured correctl. For the ARM compilers, there is an automatic
+    fallback behavior if "ARM" is the specified toolchain, if the latest compiler
+    (ARMC6) is not available, and the target supports building with both ARMC5
+    and ARMC6. In the case where the environment configuration triggers the fallback
+    to ARMC5, add a warning to the list that is returned in the results.
+
+    Returns:
+    toolchain_name - The name of the toolchain. When "ARM" is supplied as the
+    "toolchain", this be changed to either "ARMC5" or "ARMC6".
+    internal_tc_name - This corresponds to that name of the class that will be
+    used to actually complete the build. This is mostly used for accessing build
+    profiles and just general legacy sections within the code.
+    end_warnings - This is a list of warnings (strings) that were raised during
+    the process of finding toolchain. This is used to warn the user of the ARM
+    fallback mechanism mentioned above.
+
+    Positional arguments:
+    target - Target object (not the string name) of the device we are building for
+    toolchain_name - the string that identifies the build toolchain as supplied by
+    the front-end scripts
+    """
+    end_warnings = []
+    toolchain_names = get_valid_toolchain_names(target, toolchain)
+    last_error = None
+    for index, toolchain_name in enumerate(toolchain_names):
+        internal_tc_name = get_toolchain_name(target, toolchain_name)
+        if toolchain == "ARM" and toolchain_name == "ARMC5" and index != 0:
+            end_warnings.append(ARMC5_MIGRATION_WARNING)
+        if not TOOLCHAIN_CLASSES[internal_tc_name].check_executable():
+            search_path = TOOLCHAIN_PATHS[internal_tc_name] or "No path set"
+            last_error = (
+                "Could not find executable for {}.\n"
+                "Currently set search path: {}"
+            ).format(toolchain_name, search_path)
+        else:
+            return toolchain_name, internal_tc_name, end_warnings
+    else:
+        if last_error:
+            e = NoValidToolchainException(last_error)
+            e.end_warnings = end_warnings
+            raise e
+
 def get_config(src_paths, target, toolchain_name=None, app_config=None):
     """Get the configuration object for a target-toolchain combination
 
@@ -261,12 +365,18 @@ def transform_release_toolchains(target, version):
     """
     if int(target.build_tools_metadata["version"]) > 0:
         if version == '5':
+            non_arm_toolchains = set(["IAR", "GCC_ARM"])
             if 'ARMC5' in target.supported_toolchains:
-                return ['ARMC5', 'GCC_ARM', 'IAR']
+                result = ["ARMC5"]
             else:
-                return ['ARM', 'ARMC6', 'GCC_ARM', 'IAR']
-        else:
-            return target.supported_toolchains
+                result = ["ARM", "ARMC6"]
+            result.extend(
+                set(target.supported_toolchains).intersection(
+                    non_arm_toolchains
+                )
+            )
+            return result
+        return target.supported_toolchains
     else:
         if version == '5':
             return ['ARM', 'GCC_ARM', 'IAR']
@@ -315,8 +425,8 @@ def target_supports_toolchain(target, toolchain_name):
             if(toolchain_name == "ARM"):
                 #we cant find ARM, see if one ARMC5, ARMC6 or uARM listed
                 return any(tc in target.supported_toolchains for tc in ("ARMC5","ARMC6","uARM"))
-            if(toolchain_name == "ARMC6"):
-                #we did not find ARMC6, but check for ARM is listed
+            if(toolchain_name == "ARMC6" or toolchain_name == "ARMC5"):
+                #we did not find ARMC6 or ARMC5, but check if ARM is listed
                 return "ARM" in target.supported_toolchains
         #return False in other cases
         return False
@@ -1038,30 +1148,6 @@ def _lowercase_release_version(release_version):
     except AttributeError:
         return 'all'
 
-def mcu_toolchain_list(release_version='5'):
-    """  Shows list of toolchains
-
-    """
-    release_version = _lowercase_release_version(release_version)
-    version_release_targets = {}
-    version_release_target_names = {}
-
-    for version in RELEASE_VERSIONS:
-        version_release_targets[version] = get_mbed_official_release(version)
-        version_release_target_names[version] = [x[0] for x in
-                                                 version_release_targets[
-                                                     version]]
-
-    if release_version in RELEASE_VERSIONS:
-        release_targets = version_release_targets[release_version]
-    else:
-        release_targets = None
-
-    unique_supported_toolchains = get_unique_supported_toolchains(
-        release_targets)
-    columns = ["mbed OS %s" % x for x in RELEASE_VERSIONS] + unique_supported_toolchains
-    return "\n".join(columns)
-
 
 def mcu_target_list(release_version='5'):
     """  Shows target list