diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh index 7445d67ef8..bb7835cbad 100755 --- a/.github/workflows/release_prep.sh +++ b/.github/workflows/release_prep.sh @@ -37,6 +37,7 @@ mkdir -p ${PREFIX}/bazel/private cat >${INTEGRITY_FILE} < cd examples; bazel build //... @com_google_protobuf-examples-with-hyphen//... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} --enable_workspace=${{ !matrix.bzlmod }} ${{ matrix.toolchain_resolution }}; + + - name: Prebuilt test + if: ${{ matrix.bzlmod && (!matrix.continuous-only || inputs.continuous-run) }} + uses: protocolbuffers/protobuf-ci/bazel@v5 + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: examples-prebuilt-${{ matrix.bazelversion }}-${{ matrix.toolchain_resolution }} + version: ${{ matrix.bazelversion }} + bash: > + cd examples/example_without_cc_toolchain; + bazel build //... $BAZEL_FLAGS --enable_bzlmod=true ${{ matrix.toolchain_resolution }}; \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 2637b11bf9..d60a2932f2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -115,7 +115,27 @@ register_toolchains( dev_dependency = True, ) -# Proto toolchains +# Define toolchains that use pre-built protoc binaries. +prebuilt_protoc = use_extension("//bazel/private:prebuilt_protoc_extension.bzl", "protoc") +use_repo( + prebuilt_protoc, + "prebuilt_protoc.linux_aarch_64", + "prebuilt_protoc.osx_aarch_64", + "prebuilt_protoc.linux_ppcle_64", + "prebuilt_protoc.linux_s390_64", + "prebuilt_protoc.linux_x86_32", + "prebuilt_protoc.linux_x86_64", + "prebuilt_protoc.osx_x86_64", + "prebuilt_protoc.win32", + "prebuilt_protoc.win64", +) + +# However this registration only matters if the config_setting for prefer_prebuilt_protoc is true, +# using --@protobuf//bazel/toolchains:prefer_prebuilt_protoc +register_toolchains("//bazel/private/toolchains/prebuilt:all") + +# From-source protobuf toolchains +# Fallback if nothing is already registered register_toolchains("//bazel/private/toolchains:all") SUPPORTED_PYTHON_VERSIONS = [ diff --git a/bazel/private/prebuilt_protoc_extension.bzl b/bazel/private/prebuilt_protoc_extension.bzl new file mode 100644 index 0000000000..00ac1609ee --- /dev/null +++ b/bazel/private/prebuilt_protoc_extension.bzl @@ -0,0 +1,15 @@ +"Module extensions for use under bzlmod" + +load("@bazel_skylib//lib:modules.bzl", "modules") +load("//bazel/private:prebuilt_protoc_toolchain.bzl", "prebuilt_protoc_repo") +load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS") + +def create_all_toolchain_repos(name = "prebuilt_protoc"): + for platform in PROTOBUF_PLATFORMS.keys(): + prebuilt_protoc_repo( + # We must replace hyphen with underscore to workaround rules_python py_proto_library constraint + name = ".".join([name, platform.replace("-", "_")]), + platform = platform, + ) + +protoc = modules.as_extension(create_all_toolchain_repos) diff --git a/bazel/private/prebuilt_protoc_toolchain.bzl b/bazel/private/prebuilt_protoc_toolchain.bzl new file mode 100644 index 0000000000..b583e35d64 --- /dev/null +++ b/bazel/private/prebuilt_protoc_toolchain.bzl @@ -0,0 +1,57 @@ +"Repository rule that downloads a pre-compiled protoc from our official release for a single platform." + +load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS") +load(":prebuilt_tool_integrity.bzl", "RELEASED_BINARY_INTEGRITY", "RELEASE_VERSION") + +def release_version_to_artifact_name(release_version, platform): + # versions have a "v" prefix like "v28.0" + stripped_version = release_version.removeprefix("v") + + # release candidate versions like "v29.0-rc3" have artifact names + # like "protoc-29.0-rc-3-osx-x86_64.zip" + artifact_version = stripped_version.replace("rc", "rc-") + + return "{}-{}-{}.zip".format( + "protoc", + artifact_version, + platform, + ) + +def _prebuilt_protoc_repo_impl(rctx): + filename = release_version_to_artifact_name( + RELEASE_VERSION, + rctx.attr.platform, + ) + rctx.download_and_extract( + url = "https://github.com/protocolbuffers/protobuf/releases/download/{}/{}".format( + RELEASE_VERSION, + filename, + ), + sha256 = RELEASED_BINARY_INTEGRITY[filename], + ) + + rctx.file("BUILD.bazel", """\ +# Generated by @protobuf//bazel/private:prebuilt_protoc_toolchain.bzl +load("@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", "proto_toolchain") + +package(default_visibility = ["//visibility:public"]) + +proto_toolchain( + name = "prebuilt_protoc_toolchain", + proto_compiler = "{protoc_label}", +) +""".format( + protoc_label = "bin/protoc.exe" if rctx.attr.platform.startswith("win") else "bin/protoc", + )) + +prebuilt_protoc_repo = repository_rule( + doc = "Download a pre-built protoc and create a concrete toolchains for it", + implementation = _prebuilt_protoc_repo_impl, + attrs = { + "platform": attr.string( + doc = "A platform that protobuf ships a release for", + mandatory = True, + values = PROTOBUF_PLATFORMS.keys(), + ), + }, +) diff --git a/bazel/private/prebuilt_tool_integrity.bzl b/bazel/private/prebuilt_tool_integrity.bzl index 029aa8a033..913b217cff 100644 --- a/bazel/private/prebuilt_tool_integrity.bzl +++ b/bazel/private/prebuilt_tool_integrity.bzl @@ -3,18 +3,19 @@ This file contents are entirely replaced during release publishing, by .github/workflows/release_prep.sh so that the integrity of the prebuilt tools is included in the release artifact. -The checked in content is only here to allow load() statements in the sources to resolve. +The checked in content is only here to allow load() statements in the sources to resolve, and permit local testing. """ -# Create a mapping for every tool name to the hash of /dev/null -NULLSHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" -RELEASED_BINARY_INTEGRITY = { - "-".join([ - "protoc", - os, - arch, - ]): NULLSHA - for [os, arch] in { - "linux": ["aarch_64", "x86_64"], - } -} +# An arbitrary version of protobuf that includes pre-built binaries. +# See /examples/example_without_cc_toolchain which uses this for testing. +# TODO: add some automation to update this version occasionally. +_TEST_VERSION = "v33.0" +_TEST_SHAS = dict() + +# Add a couple platforms which are commonly used for testing. +_TEST_SHAS["protoc-33.0-linux-x86_64.zip"] = "d99c011b799e9e412064244f0be417e5d76c9b6ace13a2ac735330fa7d57ad8f" +_TEST_SHAS["protoc-33.0-osx-aarch_64.zip"] = "3cf55dd47118bd2efda9cd26b74f8bbbfcf5beb1bf606bc56ad4c001b543f6d3" +_TEST_SHAS["protoc-33.0-win64.zip"] = "3742cd49c8b6bd78b6760540367eb0ff62fa70a1032e15dafe131bfaf296986a" + +RELEASE_VERSION = _TEST_VERSION +RELEASED_BINARY_INTEGRITY = _TEST_SHAS diff --git a/bazel/private/proto_library_rule.bzl b/bazel/private/proto_library_rule.bzl index 8077fa3ab6..92915c2b5a 100644 --- a/bazel/private/proto_library_rule.bzl +++ b/bazel/private/proto_library_rule.bzl @@ -113,6 +113,7 @@ def _proto_library_impl(ctx): default_runfiles = ctx.runfiles(), # empty data_runfiles = data_runfiles, ), + OutputGroupInfo(_validation = ctx.attr._authenticity_validation[OutputGroupInfo]._validation), ] def _process_srcs(ctx, srcs, import_prefix, strip_import_prefix): @@ -375,6 +376,10 @@ List of files containing extension declarations. This attribute is only allowed for use with MessageSet. """, ), + "_authenticity_validation": attr.label( + default = "//bazel/private/toolchains/prebuilt:authenticity_validation", + doc = "Validate that the binary registered on the toolchain is produced by protobuf team", + ), # buildifier: disable=attr-license (calling attr.license()) "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), "_experimental_proto_descriptor_sets_include_source_info": attr.label( diff --git a/bazel/private/toolchains/prebuilt/BUILD.bazel b/bazel/private/toolchains/prebuilt/BUILD.bazel new file mode 100644 index 0000000000..e4786894de --- /dev/null +++ b/bazel/private/toolchains/prebuilt/BUILD.bazel @@ -0,0 +1,32 @@ +"""Create lazy definitions to reference the pre-built protoc toolchains. + +Ensures that Bazel only downloads required binaries for selected toolchains. +In particular, see comment below on the toolchain#toolchain attribute. +""" + +load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS") +load(":protoc_authenticity.bzl", "protoc_authenticity") + +[ + toolchain( + name = "{}_toolchain".format(platform.replace("-", "_")), + exec_compatible_with = meta["compatible_with"], + # Toolchain resolution will only permit this toolchain if the config_setting for prefer_prebuilt_protoc is true, + target_settings = ["@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc.flag_set"], + # Bazel does not follow this attribute during analysis, so the referenced repo + # will only be fetched if this toolchain is selected. + toolchain = "@prebuilt_protoc.{}//:prebuilt_protoc_toolchain".format(platform.replace("-", "_")), + toolchain_type = "@com_google_protobuf//bazel/private:proto_toolchain_type", + ) + for platform, meta in PROTOBUF_PLATFORMS.items() +] + +# Support verification of user-registered toolchains +protoc_authenticity( + name = "authenticity_validation", + fail_on_mismatch = select({ + "//bazel/toolchains:allow_nonstandard_protoc.flag_set": False, + "//conditions:default": True, + }), + visibility = ["//visibility:public"], +) diff --git a/bazel/private/toolchains/prebuilt/protoc_authenticity.bzl b/bazel/private/toolchains/prebuilt/protoc_authenticity.bzl new file mode 100644 index 0000000000..39dcae7c32 --- /dev/null +++ b/bazel/private/toolchains/prebuilt/protoc_authenticity.bzl @@ -0,0 +1,69 @@ +"Validate that the protoc binary is authentic and not spoofed by a malicious actor." + +load("//bazel/common:proto_common.bzl", "proto_common") +load("//bazel/private:prebuilt_tool_integrity.bzl", "RELEASE_VERSION") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") + +def _protoc_authenticity_impl(ctx): + # When this flag is disabled, then users have no way to replace the protoc binary with their own toolchain registration. + # Therefore there's no validation to perform. + if not proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: + return [OutputGroupInfo(_validation = depset())] + toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN] + if not toolchain: + fail("Protocol compiler toolchain could not be resolved.") + proto_lang_toolchain_info = toolchain.proto + validation_output = ctx.actions.declare_file("validation_output.txt") + + ctx.actions.run_shell( + mnemonic = "ProtocAuthenticityCheck", + outputs = [validation_output], + tools = [proto_lang_toolchain_info.proto_compiler], + command = """\ + {protoc} --version > {validation_output} + grep -q -e "-dev$" {validation_output} && {{ + echo 'WARNING: Detected a development version of protoc. + Development versions are not validated for authenticity. + To ensure a secure build, please use a released version of protoc.' + exit 0 + }} + grep -q "^libprotoc {RELEASE_VERSION}" {validation_output} || {{ + echo '{severity}: protoc version does not match protobuf Bazel module; we do not support this. + It is considered undefined behavior that is expected to break in the future even if it appears to work today.' + echo '{suppression_note}' + echo 'Expected: libprotoc {RELEASE_VERSION}' + echo -n 'Actual: ' + cat {validation_output} + exit {mismatch_exit_code} + }} >&2 + """.format( + protoc = proto_lang_toolchain_info.proto_compiler.executable.path, + validation_output = validation_output.path, + RELEASE_VERSION = RELEASE_VERSION.removeprefix("v"), + suppression_note = ( + "To suppress this error, run Bazel with --@com_google_protobuf//bazel/toolchains:allow_nonstandard_protoc" if ctx.attr.fail_on_mismatch else "" + ), + mismatch_exit_code = 1 if ctx.attr.fail_on_mismatch else 0, + severity = "ERROR" if ctx.attr.fail_on_mismatch else "INFO", + ), + ) + return [OutputGroupInfo(_validation = depset([validation_output]))] + +protoc_authenticity = rule( + implementation = _protoc_authenticity_impl, + fragments = ["proto"], + attrs = { + "fail_on_mismatch": attr.bool( + default = True, + doc = "If true, the build will fail when the protoc binary does not match the expected version.", + ), + } | toolchains.if_legacy_toolchain({ + "_proto_compiler": attr.label( + cfg = "exec", + executable = True, + allow_files = True, + default = "//src/google/protobuf/compiler:protoc_minimal", + ), + }), + toolchains = toolchains.use_toolchain(toolchains.PROTO_TOOLCHAIN), +) diff --git a/bazel/toolchains/BUILD b/bazel/toolchains/BUILD index 2f629a69e5..4ba944746a 100644 --- a/bazel/toolchains/BUILD +++ b/bazel/toolchains/BUILD @@ -1,13 +1,16 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") -package(default_applicable_licenses = ["//:license"]) +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//visibility:public"], +) bzl_library( name = "proto_toolchain_bzl", srcs = [ "proto_toolchain.bzl", ], - visibility = ["//visibility:public"], deps = [ "//bazel/private:proto_toolchain_rule_bzl", "//bazel/private:toolchain_helpers_bzl", @@ -19,7 +22,6 @@ bzl_library( srcs = [ "proto_lang_toolchain.bzl", ], - visibility = ["//visibility:public"], deps = [ "//bazel/common:proto_common_bzl", "//bazel/private:proto_lang_toolchain_rule_bzl", @@ -39,3 +41,26 @@ filegroup( "//bazel:__pkg__", ], ) + +# The public API users set +bool_flag( + name = "prefer_prebuilt_protoc", + # TODO: this should be True after the feature is vetted with some adoption + build_setting_default = False, +) + +config_setting( + name = "prefer_prebuilt_protoc.flag_set", + flag_values = {":prefer_prebuilt_protoc": "true"}, +) + +# The public API users set to disable the validation action failing. +bool_flag( + name = "allow_nonstandard_protoc", + build_setting_default = False, +) + +config_setting( + name = "allow_nonstandard_protoc.flag_set", + flag_values = {":allow_nonstandard_protoc": "true"}, +) diff --git a/examples/.gitignore b/examples/.gitignore index 229542d23c..be0a4547c0 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,2 +1,2 @@ # Ignore the bazel symlinks -/bazel-* +bazel-* diff --git a/examples/example_without_cc_toolchain/.bazelrc b/examples/example_without_cc_toolchain/.bazelrc new file mode 100644 index 0000000000..8653aa707b --- /dev/null +++ b/examples/example_without_cc_toolchain/.bazelrc @@ -0,0 +1,6 @@ +# Simulate a non-functional CC toolchain +common --per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN +common --host_per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN +# But, users should be able to use pre-built protoc toolchains instead. +common --incompatible_enable_proto_toolchain_resolution +common --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc \ No newline at end of file diff --git a/examples/example_without_cc_toolchain/BUILD.bazel b/examples/example_without_cc_toolchain/BUILD.bazel new file mode 100644 index 0000000000..ba5d829d76 --- /dev/null +++ b/examples/example_without_cc_toolchain/BUILD.bazel @@ -0,0 +1,6 @@ +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") + +proto_library( + name = "empty_proto", + srcs = ["empty.proto"], +) diff --git a/examples/example_without_cc_toolchain/MODULE.bazel b/examples/example_without_cc_toolchain/MODULE.bazel new file mode 100644 index 0000000000..666887c542 --- /dev/null +++ b/examples/example_without_cc_toolchain/MODULE.bazel @@ -0,0 +1,13 @@ +"""Bazel module dependencies""" + +module( + name = "com_google_protobuf-example-without-cc-toolchain", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "protobuf", repo_name = "com_google_protobuf") +local_path_override( + module_name = "protobuf", + path = "../..", +) diff --git a/examples/example_without_cc_toolchain/README.md b/examples/example_without_cc_toolchain/README.md new file mode 100644 index 0000000000..bcadd0af2f --- /dev/null +++ b/examples/example_without_cc_toolchain/README.md @@ -0,0 +1,3 @@ +This example demonstrates what happens when a Bazel user doesn't have a proper +CC toolchain installed. This case commonly happens in projects with no C++ code, +so they don't have a hermetic method of building C++ code. diff --git a/examples/example_without_cc_toolchain/empty.proto b/examples/example_without_cc_toolchain/empty.proto new file mode 100644 index 0000000000..1a4aede574 --- /dev/null +++ b/examples/example_without_cc_toolchain/empty.proto @@ -0,0 +1,5 @@ +edition = "2023"; + +package examples.without.cc.toolchain; + +message EmptyMessage {} diff --git a/toolchain/platforms.bzl b/toolchain/platforms.bzl new file mode 100644 index 0000000000..dbd386a104 --- /dev/null +++ b/toolchain/platforms.bzl @@ -0,0 +1,64 @@ +"List of published platforms on protobuf GitHub releases" + +# Keys are chosen to match the filenames published on protocolbuffers/protobuf releases +# NB: keys in this list are nearly identical to /toolchain/BUILD.bazel#TOOLCHAINS +# Perhaps we should share code. +PROTOBUF_PLATFORMS = { + # "k8", # this is in /toolchain/BUILD.bazel but not a released platform + # "osx-universal_binary", # this is not in /toolchain/BUILD.bazel + # but also Bazel will never request it, as we have a darwin binary for each architecture + "linux-aarch_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + }, + "linux-ppcle_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:ppc64le", + ], + }, + "linux-s390_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:s390x", + ], + }, + "linux-x86_32": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:x86_32", + ], + }, + "linux-x86_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + }, + "osx-aarch_64": { + "compatible_with": [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + }, + "osx-x86_64": { + "compatible_with": [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + }, + "win32": { + "compatible_with": [ + "@platforms//os:windows", + "@platforms//cpu:x86_32", + ], + }, + "win64": { + "compatible_with": [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + }, +}