Separate protocol compiler toolchain into new repo

mbland · mbland · commit 101494c6e123 · 2025-03-20T17:43:08.000-04:00
This is probably for the best, as it enables users to use earlier protobuf versions by not loading `//protoc:toolchains.bzl`. Leaving it wired into `scala/toolchains_repo.bzl` would've required users to patch it and to remove `protoc/BUILD`. Prompted by: #1710 (comment)
diff --git a/README.md b/README.md
@@ -76,7 +76,7 @@ host_platform_repo(name = "host_platform")
 
 # This is optional, but still safe to include even when not using
 # `--incompatible_enable_proto_toolchain_resolution`. Requires calling
-# `scala_toolchains()`.
+# `scala_protoc_toolchains()` as seen below.
 register_toolchains("@rules_scala//protoc:all")
 
 load("@rules_java//java:rules_java_deps.bzl", "rules_java_dependencies")
@@ -125,6 +125,13 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Include this after loading `platforms` and `com_google_protobuf` to enable the
+# `//protoc` precompiled protocol compiler toolchains. See the "Using a
+# precompiled protocol compiler" section below.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 # Stores the selected Scala version and other configuration parameters.
@@ -226,13 +233,37 @@ other toolchain registrations. It's safe to include even when not using
 register_toolchains("@rules_scala//protoc:all")
 ```
 
-Note that you _must_ call `scala_toolchains()` in `WORKSPACE`, even if all other
-toolchains are disabled (i.e., if using `scala_toolchains(scala = False)`). This
-isn't necessary under Bzlmod.
+#### Using `scala_protoc` in `MODULE.bazel`
+
+The `scala_protoc` extension instantiates the protocol compiler toolchain
+binaries under Bzlmod:
+
+```py
+# MODULE.bazel
+
+scala_protoc = use_extension(
+    "@rules_scala//scala/extensions:protoc.bzl",
+    "scala_protoc",
+)
+```
+
+#### Calling `scala_protoc_toolchains()` in `WORKSPACE`
+
+The `scala_protoc_toolchains` macro instantiates the protocol compiler toolchain
+binaries under `WORKSPACE`:
+
+```py
+# WORKSPACE
+
+# Include this after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+```
 
 #### Specifying additional `protoc` platforms
 
-Use the `protoc_platforms` parameter to specify additional [platforms][] if the
+Use the `platforms` parameter to specify additional [platforms][] if the
 execution platform may differ from the host platform, as when building with
 remote execution. Valid values come from the file name suffixes of
 [protocolbuffers/protobuf releases][]. It's also safe to explicitly include the
@@ -249,12 +280,8 @@ the remote execution platform is Linux running on an x86 processor.
 ```py
 # MODULE.bazel
 
-scala_deps = use_extension(
-    "@rules_scala//scala/extensions:deps.bzl",
-    "scala_deps",
-)
-scala_deps.toolchains(
-    protoc_platforms = ["linux-x86_64"],
+scala_protoc.toolchains(
+    platforms = ["linux-x86_64"],
 )
 ```
 
@@ -263,13 +290,8 @@ In `WORKSPACE` you would include:
 ```py
 # WORKSPACE
 
-load(
-    "@rules_scala//scala:toolchains.bzl",
-    "scala_toolchains",
-)
-
-scala_toolchains(
-    protoc_platforms = ["linux-x86_64"],
+scala_protoc_toolchains(
+    platforms = ["linux-x86_64"],
 )
 ```
 
@@ -936,19 +958,15 @@ builds by avoiding `@com_google_protobuf//:protoc` recompilation.
 
 ### Minimum of `protobuf` v28
 
-`rules_scala` requires at least `protobuf` v28, which contains the
-`bazel/common/proto_common.bzl` required for [protocol compiler
-toolchain](#protoc) support. This file appeared in v27, and no `ScalaPB` release
-supports `protobuf` v25.6, v26, or v27.
+`rules_scala` requires at least `protobuf` v28, and at least v29 for [protocol
+compiler toolchain](#protoc) support. No `ScalaPB` release supports `protobuf`
+v25.6, v26, or v27.
 
 #### Using earlier `protobuf` versions
 
-If you can't update to `protobuf` v28 or later right now, you will need to patch:
-
-- `scala/toolchains.bzl`: remove `setup_protoc_toolchains()` references
-- `protoc/BUILD`: remove entirely
-
-Then build using Bazel 7 and the following maximum versions of key dependencies:
+If you can't update to `protobuf` v28 or later right now, build using Bazel 7
+and the following maximum versions of key dependencies. This is not officially
+supported, but should work for some time.
 
 | Dependency | Max compatible version | Reason |
 | :-: | :-: | :- |
@@ -1114,8 +1132,8 @@ flags][protoc-opts] for protocol compiler configuration requirements.
 
 #### Using older versions of `protobuf`
 
-See [Minimum of protobuf v28](#minimum-of-protobuf-v28) for details on using
-older versions of protobuf.
+See [Using earlier protobuf versions](#using-earlier-protobuf-versions) for
+details on using older versions of protobuf if necessary.
 
 ### `scala_proto` not supported for Scala 2.11
 
diff --git a/WORKSPACE b/WORKSPACE
@@ -56,6 +56,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("//:scala_config.bzl", "scala_config")
 
 scala_config(enable_compiler_dependency_tracking = True)
diff --git a/dt_patches/test_dt_patches/WORKSPACE b/dt_patches/test_dt_patches/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(enable_compiler_dependency_tracking = True)
diff --git a/dt_patches/test_dt_patches_user_srcjar/WORKSPACE b/dt_patches/test_dt_patches_user_srcjar/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(enable_compiler_dependency_tracking = True)
diff --git a/examples/crossbuild/WORKSPACE b/examples/crossbuild/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(
diff --git a/examples/overridden_artifacts/WORKSPACE b/examples/overridden_artifacts/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(scala_version = "3.3.5")
diff --git a/examples/scala3/WORKSPACE b/examples/scala3/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(scala_version = "3.6.4")
diff --git a/examples/semanticdb/WORKSPACE b/examples/semanticdb/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(scala_version = "2.13.16")
diff --git a/examples/testing/multi_frameworks_toolchain/WORKSPACE b/examples/testing/multi_frameworks_toolchain/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config()
diff --git a/examples/testing/scalatest_repositories/WORKSPACE b/examples/testing/scalatest_repositories/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config()
diff --git a/examples/testing/specs2_junit_repositories/WORKSPACE b/examples/testing/specs2_junit_repositories/WORKSPACE
@@ -62,6 +62,11 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
 
 rules_proto_toolchains()
 
+# Must come after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+
 load("@rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config()
diff --git a/protoc/BUILD b/protoc/BUILD
@@ -6,7 +6,12 @@ load(
     "@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl",
     "proto_toolchain",
 )
-load("@rules_scala_toolchains//protoc:platforms.bzl", "PROTOC_PLATFORMS")
+load("@rules_scala_protoc_toolchains//:platforms.bzl", "PROTOC_PLATFORMS")
+
+exports_files(
+    ["toolchains.bzl"],
+    visibility = ["//visibility:public"],
+)
 
 toolchain_type(
     name = "toolchain_type",
@@ -17,13 +22,14 @@ proto_lang_toolchain(
     name = "protoc_scala_toolchain",
     command_line = "unused-because-we-pass-protoc-to-scalapb",
     toolchain_type = ":toolchain_type",
+    visibility = ["//visibility:public"],
 )
 
 [
     proto_toolchain(
         name = platform,
         exec_compatible_with = specs,
-        proto_compiler = "@rules_scala_toolchains//protoc:" + platform,
+        proto_compiler = "@rules_scala_protoc_toolchains//:" + platform,
     )
     for platform, specs in PROTOC_PLATFORMS.items()
 ]
diff --git a/protoc/private/protoc_toolchain.bzl b/protoc/private/protoc_toolchain.bzl
@@ -28,13 +28,13 @@ def _platform_build(platform):
         )
     return protoc_build
 
-def _download_build(repository_ctx, package, platform, protoc_build):
+def _download_build(repository_ctx, platform, protoc_build):
     repository_ctx.download_and_extract(
         url = PROTOC_DOWNLOAD_URL.format(
             platform = platform,
             version = PROTOC_VERSION,
         ),
-        output = "%s/%s" % (package, platform),
+        output = platform,
         integrity = protoc_build["integrity"][PROTOC_VERSION],
     )
 
@@ -47,40 +47,49 @@ def _emit_platform_entry(platform, protoc_build):
         ]),
     )
 
-def _generate_protoc_platforms(repository_ctx, package, builds):
+def _generate_protoc_platforms(repository_ctx, builds):
     content = ["PROTOC_PLATFORMS = {"]
     content.extend([_emit_platform_entry(p, b) for p, b in builds])
     content.append("}\n")
 
     repository_ctx.file(
-        package + "/platforms.bzl",
+        "platforms.bzl",
         content = "\n".join(content),
         executable = False,
     )
 
-def setup_protoc_toolchains(repository_ctx, package):
+ENABLE_PROTOC_TOOLCHAIN_ATTR = "INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION"
+
+def _scala_protoc_toolchains_impl(repository_ctx):
     builds = []
     build_file_content = ""
 
-    if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION:
+    if getattr(proto_common, ENABLE_PROTOC_TOOLCHAIN_ATTR, False):
         platforms = [_default_platform()]
-        platforms += repository_ctx.attr.protoc_platforms
+        platforms += repository_ctx.attr.platforms
         builds = {p: _platform_build(p) for p in platforms}.items()
         build_file_content = _PROTOC_TOOLCHAIN_BUILD
 
     for platform, build in builds:
-        _download_build(repository_ctx, package, platform, build)
+        _download_build(repository_ctx, platform, build)
 
-    _generate_protoc_platforms(repository_ctx, package, builds)
+    _generate_protoc_platforms(repository_ctx, builds)
 
+    # Always generate a root package, even if it's empty, to ensure
+    # `register_toolchains("@rules_scala//protoc:all")` always works.
     repository_ctx.file(
-        package + "/BUILD",
+        "BUILD",
         content = build_file_content,
         executable = False,
     )
 
-_PROTOC_TOOLCHAIN_BUILD = """
-load(":platforms.bzl", "PROTOC_PLATFORMS")
+scala_protoc_toolchains = repository_rule(
+    implementation = _scala_protoc_toolchains_impl,
+    doc = "Precompiled protocol compiler toolchain binaries",
+    attrs = {"platforms": attr.string_list()},
+)
+
+_PROTOC_TOOLCHAIN_BUILD = """load(":platforms.bzl", "PROTOC_PLATFORMS")
 
 [
     alias(
diff --git a/protoc/toolchains.bzl b/protoc/toolchains.bzl
@@ -0,0 +1,34 @@
+"""Precompiled protocol compiler toolchain repository interface.
+
+```py
+# WORKSPACE
+
+# Include this as early in the file as possible.
+register_toolchains("@rules_scala//protoc:all")
+
+# Include this after loading `platforms` and `com_google_protobuf`.
+load("@rules_scala//protoc:toolchains.bzl", "scala_protoc_toolchains")
+
+scala_protoc_toolchains()
+```
+"""
+
+load(":private/protoc_toolchain.bzl", _toolchains = "scala_protoc_toolchains")
+
+def scala_protoc_toolchains(platforms = []):
+    """Creates a repo of precompiled protocol compiler toolchain binaries.
+
+    Used by `//protoc` to implement precompiled protocol compiler toochains.
+
+    Args:
+        platforms: Operating system and architecture identifiers for
+            precompiled protocol compiler releases, taken from
+            protocolbuffers/protobuf releases file name suffixes. If
+            unspecified, will use the identifier matching the `HOST_CONSTRAINTS`
+            from `@platforms//host:constraints.bzl`. Only takes effect when
+            `--incompatible_enable_proto_toolchain_resolution` is `True`.
+    """
+    _toolchains(
+        name = "rules_scala_protoc_toolchains",
+        platforms = platforms,
+    )
diff --git a/scala/toolchains.bzl b/scala/toolchains.bzl
diff --git a/scala/toolchains_repo.bzl b/scala/toolchains_repo.bzl
diff --git a/scala_proto/scala_proto_toolchain.bzl b/scala_proto/scala_proto_toolchain.bzl
diff --git a/test_cross_build/WORKSPACE b/test_cross_build/WORKSPACE
diff --git a/test_version/WORKSPACE.template b/test_version/WORKSPACE.template
diff --git a/third_party/test/example_external_workspace/WORKSPACE b/third_party/test/example_external_workspace/WORKSPACE
diff --git a/third_party/test/proto/WORKSPACE b/third_party/test/proto/WORKSPACE
