feat(bazel): wire up prebuilt protoc toolchain (#24115)

Upstreamed from my https://github.com/aspect-build/toolchains_protoc which provided this feature outside the protobuf repo.

Adds:
- private API: module_extension + tag `protoc.prebuilt_toolchain` to provide and register toolchain for bzlmod users
- WORKSPACE users can live with the inconvenience of having to create toolchains themselves
- example of proto_library rule with intentionally non-functional cc toolchain, exercising the feature

This needs work with @thesayyn to:
- constrain users to choosing a protoc version agreeing with design doc
https://docs.google.com/document/d/16N-eU-0zHbWmxEuaUAIFTJwQLekIXZ8zIC3K4XDr20E/edit
- also register lang toolchains when the flag is enabled

Work towards #19558

Closes #24115

COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/24115 from protocolbuffers:alexeagle/use_prebuilt_toolchain b18667d82c
PiperOrigin-RevId: 844859103

.github/workflows/release_prep.sh

@@ -37,6 +37,7 @@ mkdir -p ${PREFIX}/bazel/private
 cat >${INTEGRITY_FILE} <<EOF
 "Generated during release by release_prep.sh"
 
+RELEASE_VERSION="${TAG}"
 RELEASED_BINARY_INTEGRITY = $(
 curl -s https://api.github.com/repos/protocolbuffers/protobuf/releases/tags/${TAG} \
   | jq -f <(echo "$filter_releases")

.github/workflows/test_bazel.yml

@@ -29,7 +29,12 @@ jobs:
         runner: [ ubuntu, windows, macos ]
         bazelversion: [ '7.6.1', '8.0.0' ]
         bzlmod: [ true, false ]
-        toolchain_resolution: [ "", "--incompatible_enable_proto_toolchain_resolution=true" ]
+        toolchain_resolution:
+          # Default flags, uses from-source protoc
+          - ""
+          # still uses from-source protoc unless 
+          # --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc is set
+          - "--incompatible_enable_proto_toolchain_resolution=true"
     runs-on: ${{ matrix.runner }}-latest
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Examples ${{ matrix.runner }} ${{ matrix.bazelversion }}${{ matrix.bzlmod && ' (bzlmod)' || '' }} ${{ matrix.toolchain_resolution && ' (toolchain resolution)' || '' }}
     steps:
@@ -72,3 +77,14 @@ jobs:
           bash: >
             cd examples;
             bazel build //... @com_google_protobuf-examples-with-hyphen//... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} --enable_workspace=${{ !matrix.bzlmod }} ${{ matrix.toolchain_resolution }};
+
+      - name: Prebuilt test
+        if: ${{ matrix.bzlmod && (!matrix.continuous-only || inputs.continuous-run) }}
+        uses: protocolbuffers/protobuf-ci/bazel@v5
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: examples-prebuilt-${{ matrix.bazelversion }}-${{ matrix.toolchain_resolution }}
+          version: ${{ matrix.bazelversion }}
+          bash: >
+            cd examples/example_without_cc_toolchain;
+            bazel build //... $BAZEL_FLAGS --enable_bzlmod=true ${{ matrix.toolchain_resolution }};

MODULE.bazel

@@ -115,7 +115,27 @@ register_toolchains(
     dev_dependency = True,
 )
 
-# Proto toolchains
+# Define toolchains that use pre-built protoc binaries.
+prebuilt_protoc = use_extension("//bazel/private:prebuilt_protoc_extension.bzl", "protoc")
+use_repo(
+    prebuilt_protoc,
+    "prebuilt_protoc.linux_aarch_64",
+    "prebuilt_protoc.osx_aarch_64",
+    "prebuilt_protoc.linux_ppcle_64",
+    "prebuilt_protoc.linux_s390_64",
+    "prebuilt_protoc.linux_x86_32",
+    "prebuilt_protoc.linux_x86_64",
+    "prebuilt_protoc.osx_x86_64",
+    "prebuilt_protoc.win32",
+    "prebuilt_protoc.win64",
+)
+
+# However this registration only matters if the config_setting for prefer_prebuilt_protoc is true,
+# using --@protobuf//bazel/toolchains:prefer_prebuilt_protoc
+register_toolchains("//bazel/private/toolchains/prebuilt:all")
+
+# From-source protobuf toolchains
+# Fallback if nothing is already registered
 register_toolchains("//bazel/private/toolchains:all")
 
 SUPPORTED_PYTHON_VERSIONS = [

bazel/private/prebuilt_protoc_extension.bzl

@@ -0,0 +1,15 @@
+"Module extensions for use under bzlmod"
+
+load("@bazel_skylib//lib:modules.bzl", "modules")
+load("//bazel/private:prebuilt_protoc_toolchain.bzl", "prebuilt_protoc_repo")
+load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
+
+def create_all_toolchain_repos(name = "prebuilt_protoc"):
+    for platform in PROTOBUF_PLATFORMS.keys():
+        prebuilt_protoc_repo(
+            # We must replace hyphen with underscore to workaround rules_python py_proto_library constraint
+            name = ".".join([name, platform.replace("-", "_")]),
+            platform = platform,
+        )
+
+protoc = modules.as_extension(create_all_toolchain_repos)

bazel/private/prebuilt_protoc_toolchain.bzl

@@ -0,0 +1,57 @@
+"Repository rule that downloads a pre-compiled protoc from our official release for a single platform."
+
+load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
+load(":prebuilt_tool_integrity.bzl", "RELEASED_BINARY_INTEGRITY", "RELEASE_VERSION")
+
+def release_version_to_artifact_name(release_version, platform):
+    # versions have a "v" prefix like "v28.0"
+    stripped_version = release_version.removeprefix("v")
+
+    # release candidate versions like "v29.0-rc3" have artifact names
+    # like "protoc-29.0-rc-3-osx-x86_64.zip"
+    artifact_version = stripped_version.replace("rc", "rc-")
+
+    return "{}-{}-{}.zip".format(
+        "protoc",
+        artifact_version,
+        platform,
+    )
+
+def _prebuilt_protoc_repo_impl(rctx):
+    filename = release_version_to_artifact_name(
+        RELEASE_VERSION,
+        rctx.attr.platform,
+    )
+    rctx.download_and_extract(
+        url = "https://github.com/protocolbuffers/protobuf/releases/download/{}/{}".format(
+            RELEASE_VERSION,
+            filename,
+        ),
+        sha256 = RELEASED_BINARY_INTEGRITY[filename],
+    )
+
+    rctx.file("BUILD.bazel", """\
+# Generated by @protobuf//bazel/private:prebuilt_protoc_toolchain.bzl
+load("@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", "proto_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_toolchain(
+    name = "prebuilt_protoc_toolchain",
+    proto_compiler = "{protoc_label}",
+)
+""".format(
+        protoc_label = "bin/protoc.exe" if rctx.attr.platform.startswith("win") else "bin/protoc",
+    ))
+
+prebuilt_protoc_repo = repository_rule(
+    doc = "Download a pre-built protoc and create a concrete toolchains for it",
+    implementation = _prebuilt_protoc_repo_impl,
+    attrs = {
+        "platform": attr.string(
+            doc = "A platform that protobuf ships a release for",
+            mandatory = True,
+            values = PROTOBUF_PLATFORMS.keys(),
+        ),
+    },
+)

bazel/private/prebuilt_tool_integrity.bzl

@@ -3,18 +3,19 @@
 This file contents are entirely replaced during release publishing, by .github/workflows/release_prep.sh
 so that the integrity of the prebuilt tools is included in the release artifact.
 
-The checked in content is only here to allow load() statements in the sources to resolve.
+The checked in content is only here to allow load() statements in the sources to resolve, and permit local testing.
 """
 
-# Create a mapping for every tool name to the hash of /dev/null
-NULLSHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-RELEASED_BINARY_INTEGRITY = {
-    "-".join([
-        "protoc",
-        os,
-        arch,
-    ]): NULLSHA
-    for [os, arch] in {
-        "linux": ["aarch_64", "x86_64"],
-    }
-}
+# An arbitrary version of protobuf that includes pre-built binaries.
+# See /examples/example_without_cc_toolchain which uses this for testing.
+# TODO: add some automation to update this version occasionally.
+_TEST_VERSION = "v33.0"
+_TEST_SHAS = dict()
+
+# Add a couple platforms which are commonly used for testing.
+_TEST_SHAS["protoc-33.0-linux-x86_64.zip"] = "d99c011b799e9e412064244f0be417e5d76c9b6ace13a2ac735330fa7d57ad8f"
+_TEST_SHAS["protoc-33.0-osx-aarch_64.zip"] = "3cf55dd47118bd2efda9cd26b74f8bbbfcf5beb1bf606bc56ad4c001b543f6d3"
+_TEST_SHAS["protoc-33.0-win64.zip"] = "3742cd49c8b6bd78b6760540367eb0ff62fa70a1032e15dafe131bfaf296986a"
+
+RELEASE_VERSION = _TEST_VERSION
+RELEASED_BINARY_INTEGRITY = _TEST_SHAS

bazel/private/proto_library_rule.bzl

@@ -113,6 +113,7 @@ def _proto_library_impl(ctx):
             default_runfiles = ctx.runfiles(),  # empty
             data_runfiles = data_runfiles,
         ),
+        OutputGroupInfo(_validation = ctx.attr._authenticity_validation[OutputGroupInfo]._validation),
     ]
 
 def _process_srcs(ctx, srcs, import_prefix, strip_import_prefix):
@@ -375,6 +376,10 @@ List of files containing extension declarations. This attribute is only allowed
 for use with MessageSet.
 """,
         ),
+        "_authenticity_validation": attr.label(
+            default = "//bazel/private/toolchains/prebuilt:authenticity_validation",
+            doc = "Validate that the binary registered on the toolchain is produced by protobuf team",
+        ),
         # buildifier: disable=attr-license (calling attr.license())
         "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(),
         "_experimental_proto_descriptor_sets_include_source_info": attr.label(

bazel/private/toolchains/prebuilt/BUILD.bazel

@@ -0,0 +1,32 @@
+"""Create lazy definitions to reference the pre-built protoc toolchains.
+
+Ensures that Bazel only downloads required binaries for selected toolchains.
+In particular, see comment below on the toolchain#toolchain attribute.
+"""
+
+load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
+load(":protoc_authenticity.bzl", "protoc_authenticity")
+
+[
+    toolchain(
+        name = "{}_toolchain".format(platform.replace("-", "_")),
+        exec_compatible_with = meta["compatible_with"],
+        # Toolchain resolution will only permit this toolchain if the config_setting for prefer_prebuilt_protoc is true,
+        target_settings = ["@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc.flag_set"],
+        # Bazel does not follow this attribute during analysis, so the referenced repo
+        # will only be fetched if this toolchain is selected.
+        toolchain = "@prebuilt_protoc.{}//:prebuilt_protoc_toolchain".format(platform.replace("-", "_")),
+        toolchain_type = "@com_google_protobuf//bazel/private:proto_toolchain_type",
+    )
+    for platform, meta in PROTOBUF_PLATFORMS.items()
+]
+
+# Support verification of user-registered toolchains
+protoc_authenticity(
+    name = "authenticity_validation",
+    fail_on_mismatch = select({
+        "//bazel/toolchains:allow_nonstandard_protoc.flag_set": False,
+        "//conditions:default": True,
+    }),
+    visibility = ["//visibility:public"],
+)

bazel/private/toolchains/prebuilt/protoc_authenticity.bzl

@@ -0,0 +1,69 @@
+"Validate that the protoc binary is authentic and not spoofed by a malicious actor."
+
+load("//bazel/common:proto_common.bzl", "proto_common")
+load("//bazel/private:prebuilt_tool_integrity.bzl", "RELEASE_VERSION")
+load("//bazel/private:toolchain_helpers.bzl", "toolchains")
+
+def _protoc_authenticity_impl(ctx):
+    # When this flag is disabled, then users have no way to replace the protoc binary with their own toolchain registration.
+    # Therefore there's no validation to perform.
+    if not proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION:
+        return [OutputGroupInfo(_validation = depset())]
+    toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN]
+    if not toolchain:
+        fail("Protocol compiler toolchain could not be resolved.")
+    proto_lang_toolchain_info = toolchain.proto
+    validation_output = ctx.actions.declare_file("validation_output.txt")
+
+    ctx.actions.run_shell(
+        mnemonic = "ProtocAuthenticityCheck",
+        outputs = [validation_output],
+        tools = [proto_lang_toolchain_info.proto_compiler],
+        command = """\
+        {protoc} --version > {validation_output}
+        grep -q -e "-dev$" {validation_output} && {{
+          echo 'WARNING: Detected a development version of protoc.
+          Development versions are not validated for authenticity.
+          To ensure a secure build, please use a released version of protoc.'
+          exit 0
+        }}
+        grep -q "^libprotoc {RELEASE_VERSION}" {validation_output} || {{
+          echo '{severity}: protoc version does not match protobuf Bazel module; we do not support this.
+          It is considered undefined behavior that is expected to break in the future even if it appears to work today.'
+          echo '{suppression_note}'
+          echo 'Expected: libprotoc {RELEASE_VERSION}'
+          echo -n 'Actual:   '
+          cat {validation_output}
+          exit {mismatch_exit_code}
+        }} >&2
+        """.format(
+            protoc = proto_lang_toolchain_info.proto_compiler.executable.path,
+            validation_output = validation_output.path,
+            RELEASE_VERSION = RELEASE_VERSION.removeprefix("v"),
+            suppression_note = (
+                "To suppress this error, run Bazel with --@com_google_protobuf//bazel/toolchains:allow_nonstandard_protoc" if ctx.attr.fail_on_mismatch else ""
+            ),
+            mismatch_exit_code = 1 if ctx.attr.fail_on_mismatch else 0,
+            severity = "ERROR" if ctx.attr.fail_on_mismatch else "INFO",
+        ),
+    )
+    return [OutputGroupInfo(_validation = depset([validation_output]))]
+
+protoc_authenticity = rule(
+    implementation = _protoc_authenticity_impl,
+    fragments = ["proto"],
+    attrs = {
+        "fail_on_mismatch": attr.bool(
+            default = True,
+            doc = "If true, the build will fail when the protoc binary does not match the expected version.",
+        ),
+    } | toolchains.if_legacy_toolchain({
+        "_proto_compiler": attr.label(
+            cfg = "exec",
+            executable = True,
+            allow_files = True,
+            default = "//src/google/protobuf/compiler:protoc_minimal",
+        ),
+    }),
+    toolchains = toolchains.use_toolchain(toolchains.PROTO_TOOLCHAIN),
+)

bazel/toolchains/BUILD

@@ -1,13 +1,16 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 
-package(default_applicable_licenses = ["//:license"])
+package(
+    default_applicable_licenses = ["//:license"],
+    default_visibility = ["//visibility:public"],
+)
 
 bzl_library(
     name = "proto_toolchain_bzl",
     srcs = [
         "proto_toolchain.bzl",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         "//bazel/private:proto_toolchain_rule_bzl",
         "//bazel/private:toolchain_helpers_bzl",
@@ -19,7 +22,6 @@ bzl_library(
     srcs = [
         "proto_lang_toolchain.bzl",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         "//bazel/common:proto_common_bzl",
         "//bazel/private:proto_lang_toolchain_rule_bzl",
@@ -39,3 +41,26 @@ filegroup(
         "//bazel:__pkg__",
     ],
 )
+
+# The public API users set
+bool_flag(
+    name = "prefer_prebuilt_protoc",
+    # TODO: this should be True after the feature is vetted with some adoption
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "prefer_prebuilt_protoc.flag_set",
+    flag_values = {":prefer_prebuilt_protoc": "true"},
+)
+
+# The public API users set to disable the validation action failing.
+bool_flag(
+    name = "allow_nonstandard_protoc",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "allow_nonstandard_protoc.flag_set",
+    flag_values = {":allow_nonstandard_protoc": "true"},
+)

examples/.gitignore

@@ -1,2 +1,2 @@
 # Ignore the bazel symlinks
-/bazel-*
+bazel-*

examples/example_without_cc_toolchain/.bazelrc

@@ -0,0 +1,6 @@
+# Simulate a non-functional CC toolchain
+common --per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN
+common --host_per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN
+# But, users should be able to use pre-built protoc toolchains instead.
+common --incompatible_enable_proto_toolchain_resolution
+common --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc

examples/example_without_cc_toolchain/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+
+proto_library(
+    name = "empty_proto",
+    srcs = ["empty.proto"],
+)

examples/example_without_cc_toolchain/MODULE.bazel

@@ -0,0 +1,13 @@
+"""Bazel module dependencies"""
+
+module(
+    name = "com_google_protobuf-example-without-cc-toolchain",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "protobuf", repo_name = "com_google_protobuf")
+local_path_override(
+    module_name = "protobuf",
+    path = "../..",
+)

examples/example_without_cc_toolchain/README.md

@@ -0,0 +1,3 @@
+This example demonstrates what happens when a Bazel user doesn't have a proper
+CC toolchain installed. This case commonly happens in projects with no C++ code,
+so they don't have a hermetic method of building C++ code.

examples/example_without_cc_toolchain/empty.proto

@@ -0,0 +1,5 @@
+edition = "2023";
+
+package examples.without.cc.toolchain;
+
+message EmptyMessage {}

toolchain/platforms.bzl

@@ -0,0 +1,64 @@
+"List of published platforms on protobuf GitHub releases"
+
+# Keys are chosen to match the filenames published on protocolbuffers/protobuf releases
+# NB: keys in this list are nearly identical to /toolchain/BUILD.bazel#TOOLCHAINS
+# Perhaps we should share code.
+PROTOBUF_PLATFORMS = {
+    # "k8", # this is in /toolchain/BUILD.bazel but not a released platform
+    # "osx-universal_binary", # this is not in /toolchain/BUILD.bazel
+    # but also Bazel will never request it, as we have a darwin binary for each architecture
+    "linux-aarch_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:aarch64",
+        ],
+    },
+    "linux-ppcle_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:ppc64le",
+        ],
+    },
+    "linux-s390_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:s390x",
+        ],
+    },
+    "linux-x86_32": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_32",
+        ],
+    },
+    "linux-x86_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+    "osx-aarch_64": {
+        "compatible_with": [
+            "@platforms//os:macos",
+            "@platforms//cpu:aarch64",
+        ],
+    },
+    "osx-x86_64": {
+        "compatible_with": [
+            "@platforms//os:macos",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+    "win32": {
+        "compatible_with": [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_32",
+        ],
+    },
+    "win64": {
+        "compatible_with": [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+}