Address comments on workaround for Mbed OS 2 CI build after Public (#11114)

* Modify compilation API to provide a list of paths to exclude from the build.
* `_exclude_files_from_build` becomes a static method
* Replace ternary expression with simple  `if/else` statement
* Make unit test case for dirs exclusion independent of system files

Author: hugueskamba

tools/build_api.py

@@ -1075,7 +1075,7 @@ def build_mbed_libs(target, toolchain_name, clean=False, macros=None,
         # Build Things
         notify.info("Building library %s (%s, %s)" %
                     ('MBED', target.name, toolchain_name))
-        objects = toolchain.compile_sources(
+        objects = toolchain.compile_legacy_sources(
             mbed_resources, incdirs, exclude_paths
         )
         separate_objects = []

tools/test/build_api/build_api_test.py

@@ -20,8 +20,8 @@
 from mock import patch, MagicMock
 from tools.build_api import prepare_toolchain, build_project, build_library
 from tools.regions import merge_region_list
-from tools.resources import Resources
-from tools.toolchains import TOOLCHAINS
+from tools.resources import Resources, FileRef
+from tools.toolchains import TOOLCHAINS, mbedToolchain
 from tools.notifier.mock import MockNotifier
 from tools.config import Region, Config, ConfigException
 from tools.utils import ToolException
@@ -84,6 +84,53 @@ def test_always_complete_build(self, *_):
         assert any('percent' in msg and msg['percent'] == 100.0
                    for msg in notify.messages if msg)
 
+    @patch('tools.toolchains.arm.ARM_STD.parse_dependencies',
+           return_value=["foo"])
+    @patch('tools.toolchains.mbedToolchain.need_update',
+           side_effect=[i % 2 for i in range(3000)])
+    @patch('os.mkdir')
+    @patch('tools.toolchains.mbedToolchain.dump_build_profile')
+    @patch('tools.utils.run_cmd', return_value=(b'', b'', 0))
+    def test_compile_legacy_sources_always_complete_build(self, *_):
+        """Test that compile_legacy_sources() completes."""
+        notify = MockNotifier()
+        toolchain = prepare_toolchain(self.src_paths, self.build_path, self.target,
+                                      self.toolchain_name, notify=notify)
+
+        res = Resources(MockNotifier()).scan_with_toolchain(
+            self.src_paths, toolchain)
+
+        toolchain.RESPONSE_FILES=False
+        toolchain.config_processed = True
+        toolchain.config_file = "junk"
+        toolchain.compile_legacy_sources(res)
+
+        assert any('percent' in msg and msg['percent'] == 100.0
+                   for msg in notify.messages if msg)
+
+    def test_dirs_exclusion_from_file_to_compile(self):
+        """Test that dirs can be excluded from the build."""
+        files_to_compile = [
+            FileRef(
+                name="platform/TARGET_CORTEX_M/TOOLCHAIN_ARM/except.S",
+                path="./platform/TARGET_CORTEX_M/TOOLCHAIN_ARM/except.S",
+            ),
+            FileRef(
+                name="rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/targets_irq_cm4f.S",
+                path="./rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/targets_irq_cm4f.S",
+            ),
+        ]
+        exclude_dirs = ["platform/", "drivers/", "targets/"]
+        expected_compilation_queue = [
+            FileRef(
+                name="rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/targets_irq_cm4f.S",
+                path="./rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/targets_irq_cm4f.S",
+            )
+        ]
+        compilation_queue = mbedToolchain._exclude_files_from_build(
+            files_to_compile, exclude_dirs
+        )
+        self.assertEqual(compilation_queue, expected_compilation_queue)
 
     @patch('tools.build_api.Config')
     def test_prepare_toolchain_app_config(self, mock_config_init):

tools/toolchains/mbed_toolchain.py

@@ -395,29 +395,49 @@ def get_arch_file(self, objects):
         cmd_list = (c.replace("\\", "/") for c in objects if c)
         return self.make_option_file(list(cmd_list), ".archive_files.txt")
 
+    def compile_legacy_sources(
+        self, resources, inc_dirs=None, exclude_dirs=None
+    ):
+        """Compile source files with option to exclude some directories.
+
+        This method only exists to not break API compatibility and provide a
+        way to exclude directories for Mbed OS 2 builds.
+        """
+        return self._compile_sources(
+            resources, inc_dirs=inc_dirs, exclude_dirs=exclude_dirs
+        )
+
     # THIS METHOD IS BEING CALLED BY THE MBED ONLINE BUILD SYSTEM
     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
-    def compile_sources(self, resources, inc_dirs=None, exclude_paths=None):
+    def compile_sources(self, resources, inc_dirs=None):
+        """Compile source files."""
+        return self._compile_sources(resources, inc_dirs=inc_dirs)
+
+    @staticmethod
+    def _exclude_files_from_build(files_to_compile, exclude_dirs):
+        """Remove files from dirs to be excluded for the build."""
+        return [
+            file_to_compile
+            for file_to_compile in files_to_compile
+            if all(
+                exclude_dir not in file_to_compile.path
+                for exclude_dir in exclude_dirs
+            )
+        ]
+
+    def _compile_sources(self, resources, inc_dirs=None, exclude_dirs=None):
         # Web IDE progress bar for project build
         files_to_compile = (
             resources.get_file_refs(FileType.ASM_SRC) +
             resources.get_file_refs(FileType.C_SRC) +
             resources.get_file_refs(FileType.CPP_SRC)
         )
-        # Remove files from paths to be excluded from the build and create
-        # a compilation queue.
-        compile_queue = (
-            files_to_compile
-            if not exclude_paths
-            else [
-                file_to_compile
-                for exclude_path in exclude_paths
-                for file_to_compile in files_to_compile
-                if exclude_path not in file_to_compile.path
-            ]
-        )
+        if exclude_dirs:
+            compilation_queue = self._exclude_files_from_build(files_to_compile, exclude_dirs)
+        else:
+            compilation_queue = files_to_compile
 
-        self.to_be_compiled = len(compile_queue)
+        self.to_be_compiled = len(compilation_queue)
         self.compiled = 0
 
         self.notify.cc_verbose("Macros: " + ' '.join([
@@ -447,8 +467,8 @@ def compile_sources(self, resources, inc_dirs=None, exclude_paths=None):
         self.dump_build_profile()
 
         # Sort compile queue for consistency
-        compile_queue.sort()
-        for source in compile_queue:
+        compilation_queue.sort()
+        for source in compilation_queue:
             object = self.relative_object_path(self.build_dir, source)
 
             # Queue mode (multiprocessing)