Support custom output configurations

This change adds two main features:

1. You can configure compile commands for certain targets to go into
   separate files

For embedded systems development in particular, `clangd` doesn't work
well with `compile_commands.json` generated for multiple targets. For
example, if you have a build target that runs on your host machine in
one configuration and another that runs on device in another
configuration, `compile_commands.json` will contain multiple conflicting
compile commands for the same source files. `clangd` will just use the
first one it finds, which may not be the one you want to use for code
intelligence. It's convenient to have separate compile commands files
for each target, and switch the file `clangd` uses depending on how you
want to navigate your code.

By providing the `target_file_names` argument, you can associate targets
with names. Separate compile commands files will be generated for each
of the specified targets, and will be placed in subdirectories with the
specified names. Compile commands for any targets that aren't specified
in `target_file_names` will go into the main compile commands file, just
like before.

2. You can specify a different output path

If you are generating multiple compile commands files, its preferable
not to output them into the workspace root. So you can specify a
separate output path, relative to the workspace root.

This patch doesn't change any existing behavior; if you don't add either
of the new arguments to your invocation of `refresh_compile_commands`,
everything will work exactly as it did before.

Author: Chad Norvell

refresh.template.py

@@ -1405,20 +1405,60 @@ def main():
         # End:   template filled by Bazel
     ]
 
-    compile_command_entries = []
+    target_file_names = {
+        # Begin: template filled by Bazel
+        {target_file_names}
+        # End:   template filled by Bazel
+    }
+
+    # Associates compilation database file names with lists of compile commands.
+    # __all__ is a special case: It's the "catch all" for any compile commands that aren't
+    # assigned to a specific file. If no targets are assigned to specific files (i.e., if
+    # target_file_names is empty), all compile commands will go into __all__ and end up
+    # in one file.
+    compile_command_sets = {
+        '__all__': []
+    }
+
     for (target, flags) in target_flag_pairs:
-        compile_command_entries.extend(_get_commands(target, flags))
+        # If the target has a specific file name assigned, put the compile commands in their
+        # own set, to be written to their own file.
+        if target in target_file_names:
+            target_name = target_file_names[target]
+            compile_command_sets[target_name] = list(_get_commands(target, flags))
+        # Otherwise, put them into the main file.
+        else:
+            compile_command_sets['__all__'].extend(_get_commands(target, flags))
 
-    if not compile_command_entries:
+    if len(compile_command_sets) <= 1 and len(compile_command_sets['__all__']) == 0:
         log_error(""">>> Not (over)writing compile_commands.json, since no commands were extracted and an empty file is of no use.
     There should be actionable warnings, above, that led to this.""")
         sys.exit(1)
 
+    
+    if not (root_dir := pathlib.Path({out_dir})).exists():
+        root_dir.mkdir(parents=True)
+
     # Chain output into compile_commands.json
-    with open('compile_commands.json', 'w') as output_file:
-        json.dump(
-            compile_command_entries,
-            output_file,
-            indent=2, # Yay, human readability!
-            check_circular=False # For speed.
-        )
+    for target_name in compile_command_sets:
+        # If the target doesn't have a specified file name, put it into the "catch all"
+        # compilation database.
+        if target_name == '__all__':
+            file_path = root_dir / "compile_commands.json"
+        # Otherwise, write the database to the specific target file.
+        else:
+            target_dir = root_dir / target_name
+            target_dir.mkdir(exist_ok=True)
+            file_path = target_dir / "compile_commands.json"
+
+        # This condition is only relevant to __all__. If each specified target has a specified
+        # file name, there won't be any compile commands in __all__, and we shouldn't write
+        # a file with an empty array.
+        if (len(compile_command_sets[target_name]) > 0):
+            with open(file_path, 'w') as output_file:
+                json.dump(
+                    compile_command_sets[target_name],
+                    output_file,
+                    indent=2, # Yay, human readability!
+                    check_circular=False # For speed.
+                )

refresh_compile_commands.bzl

@@ -36,6 +36,15 @@ refresh_compile_commands(
         # ^ excluding headers will speed up compile_commands.json generation *considerably* because we won't need to preprocess your code to figure out which headers you use.
         # However, if you use clangd and are looking for speed, we strongly recommend you follow the instructions below instead, since clangd is going to regularly infer the wrong commands for headers and give you lots of annoyingly unnecessary red squigglies.
 
+    # Need to create separate files for specific targets? Give those targets a name and their compile commands file will be written into a subdirectory with that name.
+        # target_file_names = {
+        #     "//:host_build": "host",
+        #     "//:target_build": "target",
+        # }
+
+    # Need to write compile commands to some directory other than the workspace root? Provide a path relative to the workspace root.
+        # out_dir = ".compile_commands"
+
     # Need things to run faster? [Either for compile_commands.json generation or clangd indexing.]
     # First: You might be able to refresh compile_commands.json slightly less often, making the current runtime okay.
         # If you're adding files, clangd should make pretty decent guesses at completions, using commands from nearby files. And if you're deleting files, there's not a problem. So you may not need to rerun refresh.py on every change to BUILD files. Instead, maybe refresh becomes something you run every so often when you can spare the time, making the current runtime okay.
@@ -62,6 +71,8 @@ load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
 def refresh_compile_commands(
         name,
         targets = None,
+        target_file_names = None,
+        out_dir = None,
         exclude_headers = None,
         exclude_external_sources = False,
         **kwargs):  # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
@@ -78,9 +89,15 @@ def refresh_compile_commands(
     elif type(targets) != "dict":  # Assume they've supplied a single string/label and wrap it
         targets = {targets: ""}
 
-    # Make any package-relative labels absolute
     targets = {
-        target if target.startswith("/") or target.startswith("@") else "{}//{}:{}".format(native.repository_name(), native.package_name(), target.removeprefix(":")): flags for target, flags in targets.items()
+        _make_label_absolute(target): flags for target, flags in targets.items()
+    }
+
+    if not target_file_names:
+        target_file_names = {}
+    
+    target_file_names = {
+        _make_label_absolute(target): file_names for target, file_names in target_file_names.items()
     }
 
     # Create a wrapper script that prints a helpful error message if the python version is too old, generated from check_python_version.template.py
@@ -89,7 +106,7 @@ def refresh_compile_commands(
 
     # Generate the core, runnable python script from refresh.template.py
     script_name = name + ".py"
-    _expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
+    _expand_template(name = script_name, labels_to_flags = targets, labels_to_file_names = target_file_names, out_dir = out_dir, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
 
     # Combine them so the wrapper calls the main script
     native.py_binary(
@@ -101,6 +118,10 @@ def refresh_compile_commands(
         **kwargs
     )
 
+def _make_label_absolute(label):
+    # Make any package-relative labels absolute
+    return label if label.startswith("/") or label.startswith("@") else "{}//{}:{}".format(native.repository_name(), native.package_name(), label.removeprefix(":"))
+
 def _expand_template_impl(ctx):
     """Inject targets of interest--and other settings--into refresh.template.py, and set it up to be run."""
     script = ctx.actions.declare_file(ctx.attr.name)
@@ -111,7 +132,9 @@ def _expand_template_impl(ctx):
         substitutions = {
             # Note, don't delete whitespace. Correctly doing multiline indenting.
             "        {target_flag_pairs}": "\n".join(["        {},".format(pair) for pair in ctx.attr.labels_to_flags.items()]),
+            "        {target_file_names}": "\n".join(["        '{}': '{}',".format(target, file_name) for (target, file_name) in ctx.attr.labels_to_file_names.items()]),
             "        {windows_default_include_paths}": "\n".join(["        %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]),  # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
+            "{out_dir}": repr(ctx.attr.out_dir),
             "{exclude_headers}": repr(ctx.attr.exclude_headers),
             "{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
             "{print_args_executable}": repr(ctx.executable._print_args_executable.path),
@@ -122,6 +145,8 @@ def _expand_template_impl(ctx):
 _expand_template = rule(
     attrs = {
         "labels_to_flags": attr.string_dict(mandatory = True),  # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
+        "labels_to_file_names": attr.string_dict(),
+        "out_dir": attr.string(default = "."),
         "exclude_external_sources": attr.bool(default = False),
         "exclude_headers": attr.string(values = ["all", "external", ""]),  # "" needed only for compatibility with Bazel < 3.6.0
         "_script_template": attr.label(allow_single_file = True, default = "refresh.template.py"),