From c5a405c0b7eb7db976447b7ea04368dd19a3a96d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 10 Jun 2026 11:21:46 -0700 Subject: [PATCH] Add Kythe indexing support to HPB proto in C++ PiperOrigin-RevId: 929969615 --- hpb/bazel/hpb_proto_library.bzl | 77 +++++++- hpb_generator/BUILD | 5 +- hpb_generator/context.h | 6 +- hpb_generator/gen_accessors.cc | 114 ++++++++---- hpb_generator/gen_messages.cc | 2 +- hpb_generator/gen_repeated_fields.cc | 90 +++++++--- hpb_generator/generator.cc | 27 ++- hpb_generator/tests/BUILD | 19 ++ hpb_generator/tests/metadata_test.cc | 170 ++++++++++++++++++ .../cc_library_func.bzl | 5 +- 10 files changed, 434 insertions(+), 81 deletions(-) create mode 100644 hpb_generator/tests/metadata_test.cc diff --git a/hpb/bazel/hpb_proto_library.bzl b/hpb/bazel/hpb_proto_library.bzl index 6ed9f6d7a8..95fc8ad571 100644 --- a/hpb/bazel/hpb_proto_library.bzl +++ b/hpb/bazel/hpb_proto_library.bzl @@ -9,12 +9,14 @@ - hpb_proto_library() """ +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "use_cpp_toolchain") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//bazel/common:proto_common.bzl", "proto_common") load("//bazel/common:proto_info.bzl", "ProtoInfo") -load("//upb/bazel:upb_proto_library.bzl", "GeneratedSrcsInfo", "UpbWrappedCcInfo", "upb_proto_library_aspect") +load("//bazel/common:proto_lang_toolchain_info.bzl", "ProtoLangToolchainInfo") +load("//upb/bazel:upb_proto_library.bzl", "UpbWrappedCcInfo", "upb_proto_library_aspect") load("//upb/bazel/private:upb_proto_library_internal/cc_library_func.bzl", "cc_library_func") # buildifier: disable=bzl-visibility def upb_use_cpp_toolchain(): @@ -26,12 +28,40 @@ def _filter_none(elems): _HpbWrappedCcInfo = provider("Provider for cc_info for hpb", fields = ["cc_info"]) _WrappedCcGeneratedSrcsInfo = provider("Provider for generated sources", fields = ["srcs"]) +HpbGeneratedSrcsInfo = provider( + "Provides generated headers, sources and metadata for hpb", + fields = ["srcs", "hdrs", "metadata"], +) + def _get_lang_toolchain(ctx): return ctx.attr._hpb_lang_toolchain[proto_common.ProtoLangToolchainInfo] -def _compile_hpb_protos(ctx, proto_info, proto_sources): +def _add_annotate_headers_to_flag(out_flag): + eq_idx = out_flag.find("=") + if eq_idx == -1: + return out_flag + + param_name = out_flag[:eq_idx] + value = out_flag[eq_idx + 1:] + + colon_idx = value.rfind(":") + if colon_idx == -1: + args = "" + output = value + else: + args = value[:colon_idx] + output = value[colon_idx + 1:] + + if args: + new_args = args + ",annotate_headers" + else: + new_args = "annotate_headers" + + return param_name + "=" + new_args + ":" + output + +def _compile_hpb_protos(ctx, proto_info, proto_sources, generate_meta_data): if len(proto_sources) == 0: - return GeneratedSrcsInfo(srcs = [], hdrs = []) + return HpbGeneratedSrcsInfo(srcs = [], hdrs = [], metadata = []) srcs = [] srcs += proto_common.declare_generated_files( @@ -47,15 +77,42 @@ def _compile_hpb_protos(ctx, proto_info, proto_sources): proto_info = proto_info, ) + metadata = [] + if generate_meta_data: + metadata = proto_common.declare_generated_files( + ctx.actions, + extension = ".hpb.h.meta", + proto_info = proto_info, + ) + + toolchain = _get_lang_toolchain(ctx) + if generate_meta_data: + out_flag = _add_annotate_headers_to_flag(toolchain.out_replacement_format_flag) + + toolchain = ProtoLangToolchainInfo( + out_replacement_format_flag = out_flag, + output_files = getattr(toolchain, "output_files", "legacy"), + plugin_format_flag = toolchain.plugin_format_flag, + plugin = toolchain.plugin, + runtime = toolchain.runtime, + provided_proto_sources = toolchain.provided_proto_sources, + proto_compiler = toolchain.proto_compiler, + protoc_opts = toolchain.protoc_opts, + progress_message = toolchain.progress_message, + mnemonic = toolchain.mnemonic, + allowlist_different_package = getattr(toolchain, "allowlist_different_package", None), + toolchain_type = getattr(toolchain, "toolchain_type", None), + ) + proto_common.compile( actions = ctx.actions, proto_info = proto_info, - proto_lang_toolchain_info = _get_lang_toolchain(ctx), - generated_files = srcs + hdrs, + proto_lang_toolchain_info = toolchain, + generated_files = srcs + hdrs + metadata, experimental_exec_group = "proto_compiler", ) - return GeneratedSrcsInfo(srcs = srcs, hdrs = hdrs) + return HpbGeneratedSrcsInfo(srcs = srcs, hdrs = hdrs, metadata = metadata) def _hpb_proto_rule_impl(ctx): if len(ctx.attr.deps) != 1: @@ -80,7 +137,7 @@ def _hpb_proto_rule_impl(ctx): lib.dynamic_library, ]) return [ - DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs)), + DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs + srcs.metadata)), srcs, cc_info, ] @@ -89,6 +146,7 @@ def _get_proto_deps(ctx): return [dep for dep in ctx.rule.attr.deps if ProtoInfo in dep] def _upb_cc_proto_aspect_impl(target, ctx, cc_provider, file_provider): + generate_meta_data = False deps = _get_proto_deps(ctx) + ctx.attr._upbprotos dep_ccinfos = [dep[CcInfo] for dep in deps if CcInfo in dep] dep_ccinfos += [dep[UpbWrappedCcInfo].cc_info for dep in deps if UpbWrappedCcInfo in dep] @@ -104,9 +162,9 @@ def _upb_cc_proto_aspect_impl(target, ctx, cc_provider, file_provider): # https://bazel.build/versions/6.4.0/reference/be/protocol-buffer#proto_library.srcs return [cc_provider( cc_info = cc_common.merge_cc_infos(direct_cc_infos = dep_ccinfos), - ), file_provider(srcs = GeneratedSrcsInfo(srcs = [], hdrs = []))] + ), file_provider(srcs = HpbGeneratedSrcsInfo(srcs = [], hdrs = [], metadata = []))] else: - files = _compile_hpb_protos(ctx, proto_info, proto_info.direct_sources) + files = _compile_hpb_protos(ctx, proto_info, proto_info.direct_sources, generate_meta_data) cc_info = cc_library_func( ctx = ctx, name = ctx.rule.attr.name + "_hpb", @@ -114,6 +172,7 @@ def _upb_cc_proto_aspect_impl(target, ctx, cc_provider, file_provider): srcs = files.srcs, copts = [], dep_ccinfos = dep_ccinfos, + non_compilation_additional_inputs = files.metadata, ) return [cc_provider(cc_info = cc_info), file_provider(srcs = files)] diff --git a/hpb_generator/BUILD b/hpb_generator/BUILD index 27aaf69c69..5ecea80584 100644 --- a/hpb_generator/BUILD +++ b/hpb_generator/BUILD @@ -59,7 +59,7 @@ cc_library( hdrs = [ "generator.h", ], - visibility = ["//visibility:private"], + visibility = ["//hpb_generator:__subpackages__"], deps = [ ":context", ":gen_utils", @@ -69,6 +69,7 @@ cc_library( "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:code_generator_lite", "//src/google/protobuf/compiler:plugin", + "//src/google/protobuf/io:printer", "//upb_generator:common", "//upb_generator:file_layout", "//upb_generator/c:names", @@ -108,7 +109,7 @@ cc_library( name = "names", srcs = ["names.cc"], hdrs = ["names.h"], - visibility = ["//visibility:private"], + visibility = ["//hpb_generator:__subpackages__"], deps = [ ":context", ":keywords", diff --git a/hpb_generator/context.h b/hpb_generator/context.h index 3c2e7b0664..14222060f4 100644 --- a/hpb_generator/context.h +++ b/hpb_generator/context.h @@ -34,6 +34,7 @@ enum class Backend { UPB, CPP }; struct Options { Backend backend = Backend::UPB; bool strip_feature_includes = false; + io::AnnotationCollector* annotation_collector = nullptr; }; /** @@ -52,7 +53,10 @@ class Context final { public: Context(const FileDescriptor* file, io::ZeroCopyOutputStream* stream, const Options& options) - : stream_(stream), printer_(stream_), options_(options) { + : stream_(stream), + printer_(stream_, + io::Printer::Options('$', options.annotation_collector)), + options_(options) { BuildDefPool(file); } diff --git a/hpb_generator/gen_accessors.cc b/hpb_generator/gen_accessors.cc index 9ba5e6cead..44534f9194 100644 --- a/hpb_generator/gen_accessors.cc +++ b/hpb_generator/gen_accessors.cc @@ -21,6 +21,7 @@ #include "hpb_generator/keywords.h" #include "hpb_generator/names.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/io/printer.h" #include "upb_generator/c/names.h" #include "upb_generator/minitable/names.h" @@ -30,6 +31,7 @@ namespace hpb_generator { using NameToFieldDescriptorMap = absl::flat_hash_map; +using Sub = ::google::protobuf::io::Printer::Sub; void WriteFieldAccessorHazzer(const google::protobuf::Descriptor* desc, const google::protobuf::FieldDescriptor* field, @@ -100,20 +102,31 @@ void WriteFieldAccessorsInHeader(const google::protobuf::Descriptor* desc, Conte } else { // non-repeated. if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_STRING) { - ctx.Emit({{"field_name", resolved_field_name}}, - R"cc( - absl::string_view $field_name$() const; - void set_$field_name$(absl::string_view value); - )cc"); + ctx.Emit( + {Sub("field_name", resolved_field_name) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("set_field_name", absl::StrCat("set_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet})}, + R"cc( + absl::string_view $field_name$() const; + void $set_field_name$(absl::string_view value); + )cc"); } else if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { ctx.Emit( {{"mut_ptr_type", MessagePtrConstType(field, /* const */ false)}, {"const_ptr_type", MessagePtrConstType(field, /* const */ true)}, - {"field_name", resolved_field_name}}, + Sub("field_name", resolved_field_name) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("mutable_field_name", + absl::StrCat("mutable_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kAlias}), + Sub("set_alias_field_name", + absl::StrCat("set_alias_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet})}, R"cc( $const_ptr_type$ $field_name$() const; - $mut_ptr_type$ mutable_$field_name$(); + $mut_ptr_type$ $mutable_field_name$(); /** * Re-points submessage to the given target. * @@ -121,22 +134,26 @@ void WriteFieldAccessorsInHeader(const google::protobuf::Descriptor* desc, Conte * - both messages must be in the same arena, or in two * fused arenas. */ - void set_alias_$field_name$($mut_ptr_type$ target); + void $set_alias_field_name$($mut_ptr_type$ target); )cc"); } else { - ctx.Emit({{"cpp_type", CppConstType(field)}, - {"field_name", resolved_field_name}, - {"upb_msg_name", - upb::generator::CApiMessageType(desc->full_name())}, - {"upb_field_name", resolved_upbc_name}}, - R"cc( - inline $cpp_type$ $field_name$() const { - return $upb_msg_name$_$upb_field_name$(msg_); - } - inline void set_$field_name$($cpp_type$ value) { - return $upb_msg_name$_set_$upb_field_name$(msg_, value); - } - )cc"); + ctx.Emit( + {{"cpp_type", CppConstType(field)}, + Sub("field_name", resolved_field_name) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("set_field_name", absl::StrCat("set_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + {"upb_msg_name", + upb::generator::CApiMessageType(desc->full_name())}, + {"upb_field_name", resolved_upbc_name}}, + R"cc( + inline $cpp_type$ $field_name$() const { + return $upb_msg_name$_$upb_field_name$(msg_); + } + inline void $set_field_name$($cpp_type$ value) { + return $upb_msg_name$_set_$upb_field_name$(msg_, value); + } + )cc"); } } } @@ -151,11 +168,12 @@ void WriteFieldAccessorHazzer(const google::protobuf::Descriptor* desc, if (field->has_presence()) { // Has presence. ctx.Emit( - {{"field_name", resolved_field_name}, + {Sub("has_field_name", absl::StrCat("has_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), {"upb_msg_name", upb::generator::CApiMessageType(desc->full_name())}, {"upb_field_name", resolved_upbc_name}}, R"cc( - inline bool has_$field_name$() const { + inline bool $has_field_name$() const { return $upb_msg_name$_has_$upb_field_name$(msg_); } )cc"); @@ -169,11 +187,12 @@ void WriteFieldAccessorClear(const google::protobuf::Descriptor* desc, Context& ctx) { if (field->has_presence()) { ctx.Emit( - {{"field_name", resolved_field_name}, + {Sub("clear_field_name", absl::StrCat("clear_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), {"upb_field_name", resolved_upbc_name}, {"upb_msg_name", upb::generator::CApiMessageType(desc->full_name())}}, R"cc( - void clear_$field_name$() { + void $clear_field_name$() { $upb_msg_name$_clear_$upb_field_name$(msg_); } )cc"); @@ -189,41 +208,58 @@ void WriteMapFieldAccessors(const google::protobuf::Descriptor* desc, const google::protobuf::FieldDescriptor* key = entry->FindFieldByNumber(1); const google::protobuf::FieldDescriptor* val = entry->FindFieldByNumber(2); ctx.Emit( - {{"field_name", resolved_field_name}, + {Sub("field_name_size", absl::StrCat(resolved_field_name, "_size")) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("clear_field_name", absl::StrCat("clear_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("delete_field_name", absl::StrCat("delete_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), {"upb_msg_name", upb::generator::CApiMessageType(desc->full_name())}, {"const_key", CppConstType(key)}, {"upb_field_name", resolved_upbc_name}}, R"cc( - inline size_t $field_name$_size() const { + inline size_t $field_name_size$() const { return $upb_msg_name$_$upb_field_name$_size(msg_); } - inline void clear_$field_name$() { + inline void $clear_field_name$() { $upb_msg_name$_clear_$upb_field_name$(msg_); } - void delete_$field_name$($const_key$ key); + void $delete_field_name$($const_key$ key); )cc"); if (val->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { - ctx.Emit({{"field_name", resolved_field_name}, + ctx.Emit({Sub("set_field_name", absl::StrCat("set_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("set_alias_field_name", + absl::StrCat("set_alias_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("get_field_name", absl::StrCat("get_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("get_mutable_field_name", + absl::StrCat("get_mutable_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kAlias}), {"const_key", CppConstType(key)}, {"const_val", CppConstType(val)}, {"ConstPtr", MessagePtrConstType(val, true)}, {"MutPtr", MessagePtrConstType(val, false)}}, R"cc( - bool set_$field_name$($const_key$ key, $ConstPtr$ value); - bool set_$field_name$($const_key$ key, $MutPtr$ value); - bool set_alias_$field_name$($const_key$ key, $ConstPtr$ value); - bool set_alias_$field_name$($const_key$ key, $MutPtr$ value); - absl::StatusOr<$ConstPtr$> get_$field_name$($const_key$ key); - absl::StatusOr<$MutPtr$> get_mutable_$field_name$($const_key$ key); + bool $set_field_name$($const_key$ key, $ConstPtr$ value); + bool $set_field_name$($const_key$ key, $MutPtr$ value); + bool $set_alias_field_name$($const_key$ key, $ConstPtr$ value); + bool $set_alias_field_name$($const_key$ key, $MutPtr$ value); + absl::StatusOr<$ConstPtr$> $get_field_name$($const_key$ key); + absl::StatusOr<$MutPtr$> $get_mutable_field_name$($const_key$ key); )cc"); } else { - ctx.Emit({{"field_name", resolved_field_name}, + ctx.Emit({Sub("set_field_name", absl::StrCat("set_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("get_field_name", absl::StrCat("get_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), {"const_key", CppConstType(key)}, {"const_val", CppConstType(val)}}, R"cc( - bool set_$field_name$($const_key$ key, $const_val$ value); - absl::StatusOr<$const_val$> get_$field_name$($const_key$ key); + bool $set_field_name$($const_key$ key, $const_val$ value); + absl::StatusOr<$const_val$> $get_field_name$($const_key$ key); )cc"); } } diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index 023bb90f66..c538cbade7 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -200,7 +200,7 @@ void WriteModelPublicDeclaration( const std::vector& file_exts, const std::vector& file_enums, Context& ctx) { - ctx.Emit({{"class_name", ClassName(descriptor)}, + ctx.Emit({Sub("class_name", ClassName(descriptor)).AnnotatedAs(descriptor), {"qualified_class_name", QualifiedClassName(descriptor)}}, R"cc( class $class_name$ final : private internal::$class_name$Access { diff --git a/hpb_generator/gen_repeated_fields.cc b/hpb_generator/gen_repeated_fields.cc index c8bc0432a9..a723ac5c4a 100644 --- a/hpb_generator/gen_repeated_fields.cc +++ b/hpb_generator/gen_repeated_fields.cc @@ -11,6 +11,7 @@ #include #include "google/protobuf/descriptor.pb.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "hpb_generator/context.h" #include "hpb_generator/gen_accessors.h" @@ -19,6 +20,7 @@ #include "hpb_generator/gen_utils.h" #include "hpb_generator/names.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/io/printer.h" #include "upb_generator/c/names.h" #include "upb_generator/common.h" #include "upb_generator/file_layout.h" @@ -27,6 +29,8 @@ namespace google { namespace protobuf { namespace hpb_generator { +using Sub = ::google::protobuf::io::Printer::Sub; + // Adds using accessors to reuse base Access class members from a Proxy/CProxy. void WriteRepeatedFieldUsingAccessors(const google::protobuf::FieldDescriptor* field, absl::string_view class_name, @@ -73,10 +77,11 @@ void WriteRepeatedFieldsInMessageHeader(const google::protobuf::Descriptor* desc Context& ctx) { ctx.Emit( {{"upb_msg_name", upb::generator::CApiMessageType(desc->full_name())}, - {"field_name", resolved_field_name}, + Sub("field_name_size", absl::StrCat(resolved_field_name, "_size")) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), {"upbc_name", resolved_upbc_name}}, R"cc( - inline size_t $field_name$_size() const { + inline size_t $field_name_size$() const { size_t len; $upb_msg_name$_$upbc_name$(msg_, &len); return len; @@ -87,44 +92,75 @@ void WriteRepeatedFieldsInMessageHeader(const google::protobuf::Descriptor* desc ctx.Emit( {{"mut_ptr_type", MessagePtrConstType(field, /* const */ false)}, {"const_ptr_type", MessagePtrConstType(field, /* const */ true)}, - {"field_name", resolved_field_name}, + Sub("field_name", resolved_field_name) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("mutable_field_name", + absl::StrCat("mutable_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kAlias}), + Sub("add_field_name", absl::StrCat("add_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("add_alias_field_name", + absl::StrCat("add_alias_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), {"upbc_name", resolved_upbc_name}, {"msg_base_type", MessageBaseType(field, /* maybe_const */ false)}}, R"cc( $const_ptr_type$ $field_name$(size_t index) const; const ::hpb::RepeatedField::CProxy $field_name$() const; - ::hpb::Ptr<::hpb::RepeatedField<$msg_base_type$>> mutable_$field_name$(); - absl::StatusOr<$mut_ptr_type$> add_$field_name$(); + ::hpb::Ptr<::hpb::RepeatedField<$msg_base_type$>> $mutable_field_name$(); + absl::StatusOr<$mut_ptr_type$> $add_field_name$(); /** * Re-points submsg of repeated field to given target. * * REQUIRES: both messages must be in the same arena. */ - bool add_alias_$field_name$($mut_ptr_type$ target); - $mut_ptr_type$ mutable_$field_name$(size_t index) const; + bool $add_alias_field_name$($mut_ptr_type$ target); + $mut_ptr_type$ $mutable_field_name$(size_t index) const; )cc"); } else if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_STRING) { - ctx.Emit({{"cpp_const_type", CppConstType(field)}, - {"field_name", resolved_field_name}}, - R"cc( - $cpp_const_type$ $field_name$(size_t index) const; - const ::hpb::RepeatedField<$cpp_const_type$>::CProxy $field_name$() const; - ::hpb::Ptr<::hpb::RepeatedField<$cpp_const_type$>> mutable_$field_name$(); - bool add_$field_name$($cpp_const_type$ val); - void set_$field_name$(size_t index, $cpp_const_type$ val); - bool resize_$field_name$(size_t len); - )cc"); + ctx.Emit( + {{"cpp_const_type", CppConstType(field)}, + Sub("field_name", resolved_field_name) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("mutable_field_name", + absl::StrCat("mutable_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kAlias}), + Sub("add_field_name", absl::StrCat("add_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("set_field_name", absl::StrCat("set_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("resize_field_name", absl::StrCat("resize_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone})}, + R"cc( + $cpp_const_type$ $field_name$(size_t index) const; + const ::hpb::RepeatedField<$cpp_const_type$>::CProxy $field_name$() const; + ::hpb::Ptr<::hpb::RepeatedField<$cpp_const_type$>> $mutable_field_name$(); + bool $add_field_name$($cpp_const_type$ val); + void $set_field_name$(size_t index, $cpp_const_type$ val); + bool $resize_field_name$(size_t len); + )cc"); } else { - ctx.Emit({{"cpp_const_type", CppConstType(field)}, - {"field_name", resolved_field_name}}, - R"cc( - $cpp_const_type$ $field_name$(size_t index) const; - const ::hpb::RepeatedField<$cpp_const_type$>::CProxy $field_name$() const; - ::hpb::Ptr<::hpb::RepeatedField<$cpp_const_type$>> mutable_$field_name$(); - bool add_$field_name$($cpp_const_type$ val); - void set_$field_name$(size_t index, $cpp_const_type$ val); - bool resize_$field_name$(size_t len); - )cc"); + ctx.Emit( + {{"cpp_const_type", CppConstType(field)}, + Sub("field_name", resolved_field_name) + .AnnotatedAs({field, io::AnnotationCollector::kNone}), + Sub("mutable_field_name", + absl::StrCat("mutable_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kAlias}), + Sub("add_field_name", absl::StrCat("add_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("set_field_name", absl::StrCat("set_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kSet}), + Sub("resize_field_name", absl::StrCat("resize_", resolved_field_name)) + .AnnotatedAs({field, io::AnnotationCollector::kNone})}, + R"cc( + $cpp_const_type$ $field_name$(size_t index) const; + const ::hpb::RepeatedField<$cpp_const_type$>::CProxy $field_name$() const; + ::hpb::Ptr<::hpb::RepeatedField<$cpp_const_type$>> $mutable_field_name$(); + bool $add_field_name$($cpp_const_type$ val); + void $set_field_name$(size_t index, $cpp_const_type$ val); + bool $resize_field_name$(size_t len); + )cc"); } } diff --git a/hpb_generator/generator.cc b/hpb_generator/generator.cc index 2ccdca7f68..4275c2cef7 100644 --- a/hpb_generator/generator.cc +++ b/hpb_generator/generator.cc @@ -12,6 +12,7 @@ #include #include +#include "google/protobuf/descriptor.pb.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "google/protobuf/compiler/code_generator.h" @@ -329,6 +330,7 @@ bool Generator::Generate(const google::protobuf::FileDescriptor* file, std::string* error) const { { bool strip_nonfunctional_codegen = false; + bool annotate_headers = false; Backend backend = Backend::UPB; std::vector> params; google::protobuf::compiler::ParseGeneratorParameter(parameter, ¶ms); @@ -338,14 +340,27 @@ bool Generator::Generate(const google::protobuf::FileDescriptor* file, strip_nonfunctional_codegen = true; } else if (pair.first == "backend" && pair.second == "cpp") { backend = Backend::CPP; + } else if (pair.first == "annotate_headers") { + annotate_headers = true; } else { *error = "Unknown parameter: " + pair.first; return false; } } + + std::unique_ptr annotations; + std::unique_ptr annotation_collector; + if (annotate_headers) { + annotations = std::make_unique(); + annotation_collector = + std::make_unique>( + annotations.get()); + } + // Write model.hpb.h Options options = {.backend = backend, - .strip_feature_includes = strip_nonfunctional_codegen}; + .strip_feature_includes = strip_nonfunctional_codegen, + .annotation_collector = annotation_collector.get()}; std::unique_ptr header_output_stream( context->Open(CppHeaderFilename(file))); Context hdr_ctx(file, header_output_stream.get(), options); @@ -356,6 +371,16 @@ bool Generator::Generate(const google::protobuf::FileDescriptor* file, context->Open(CppSourceFilename(file))); auto cc_ctx = Context(file, cc_output_stream.get(), options); WriteSource(file, cc_ctx); + + if (annotate_headers) { + std::string meta_filename = + absl::StrCat(compiler::StripProto(file->name()), ".hpb.h.meta"); + std::unique_ptr meta_output_stream( + context->Open(meta_filename)); + ABSL_CHECK( + annotations->SerializeToZeroCopyStream(meta_output_stream.get())); + } + return true; } } diff --git a/hpb_generator/tests/BUILD b/hpb_generator/tests/BUILD index 1fa9a58c23..8bf854df65 100644 --- a/hpb_generator/tests/BUILD +++ b/hpb_generator/tests/BUILD @@ -257,3 +257,22 @@ cc_test( "@googletest//:gtest_main", ], ) + +cc_test( + name = "metadata_test", + srcs = ["metadata_test.cc"], + deps = [ + "//hpb_generator:generator", + "//src/google/protobuf", + "//src/google/protobuf/compiler:annotation_test_util", + "//src/google/protobuf/compiler:code_generator_lite", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/testing", + "//src/google/protobuf/testing:file", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) diff --git a/hpb_generator/tests/metadata_test.cc b/hpb_generator/tests/metadata_test.cc new file mode 100644 index 0000000000..a6ddc805aa --- /dev/null +++ b/hpb_generator/tests/metadata_test.cc @@ -0,0 +1,170 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include +#include + +#include "google/protobuf/testing/file.h" +#include "google/protobuf/testing/file.h" +#include +#include +#include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/compiler/annotation_test_util.h" +#include "google/protobuf/compiler/code_generator_lite.h" +#include "google/protobuf/compiler/command_line_interface.h" +#include "hpb_generator/generator.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace hpb { + +namespace atu = annotation_test_util; + +namespace { + +using ::testing::IsEmpty; +using Annotation = GeneratedCodeInfo::Annotation; + +class HpbMetadataTest : public ::testing::Test { + public: + // Tries to capture a FileDescriptorProto, GeneratedCodeInfo, and output + // code from the previously added file with name `filename`. Returns true on + // success. + bool CaptureMetadata(const std::string& filename, FileDescriptorProto& file, + std::string& hpb_h, GeneratedCodeInfo& hpb_h_info) { + CommandLineInterface cli; + hpb_generator::Generator hpb_generator; + cli.RegisterGenerator("--hpb_out", &hpb_generator, ""); + std::string hpb_out = + absl::StrCat("--hpb_out=annotate_headers=true:", ::testing::TempDir()); + + const bool result = atu::RunProtoCompiler(filename, hpb_out, &cli, &file); + + if (!result) { + return result; + } + + std::string output_base = + absl::StrCat(::testing::TempDir(), "/", StripProto(filename)); + + ABSL_CHECK_OK(File::GetContents(absl::StrCat(output_base, ".hpb.h"), &hpb_h, + true)); + if (!atu::DecodeMetadata(absl::StrCat(output_base, ".hpb.h.meta"), + &hpb_h_info)) { + return false; + } + + return true; + } + + // Helper function to get annotations by the provided path and verify that + // these annotations contain a subset of the expected annotations (by + // substring) + semantics. + void ExpectAnnotationsForPathContain( + const GeneratedCodeInfo& info, const std::string& filename, + const std::string& hpb_h, const std::vector& path, + const absl::flat_hash_map& + expected_annotations) { + absl::flat_hash_map + expected_annotations_copy = expected_annotations; + std::vector annotations; + atu::FindAnnotationsOnPath(info, filename, path, &annotations); + EXPECT_TRUE(!annotations.empty()); + for (const auto* annotation : annotations) { + auto substring = atu::GetAnnotationSubstring(hpb_h, *annotation); + ASSERT_TRUE(substring.has_value()); + if (auto node = expected_annotations_copy.extract(*substring); + !node.empty()) { + EXPECT_EQ(node.mapped(), annotation->semantic()) + << "for substring " << *substring; + } + } + EXPECT_THAT(expected_annotations_copy, IsEmpty()) + << "substrings above were not found in the annotations."; + } +}; + +constexpr absl::string_view kSmallTestFile = R"schema(syntax = "proto2"; +package foo; +message Message { } +)schema"; + +TEST_F(HpbMetadataTest, CapturesMessageNames) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string hpb_h; + atu::AddFile("test.proto", kSmallTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", file, hpb_h, info)); + EXPECT_EQ("Message", file.message_type(0).name()); + std::vector message_path; + message_path.push_back(FileDescriptorProto::kMessageTypeFieldNumber); + message_path.push_back(0); + const GeneratedCodeInfo::Annotation* message_annotation = + atu::FindAnnotationOnPath(info, "test.proto", message_path); + EXPECT_TRUE(nullptr != message_annotation); + EXPECT_TRUE( + atu::AnnotationMatchesSubstring(hpb_h, message_annotation, "Message")); +} + +constexpr absl::string_view kStringFieldTestFile = R"schema( + syntax = "proto2"; + package foo; + message Message { + optional string sfield = 1; + repeated string rsfield = 2; + } +)schema"; + +TEST_F(HpbMetadataTest, AnnotatesStringSemantics) { + FileDescriptorProto file; + GeneratedCodeInfo info; + std::string hpb_h; + atu::AddFile("test.proto", kStringFieldTestFile); + EXPECT_TRUE(CaptureMetadata("test.proto", file, hpb_h, info)); + EXPECT_EQ("Message", file.message_type(0).name()); + + // Check annotations for `sfield`. + std::vector field_path{ + FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kFieldFieldNumber, + 0, + }; + ExpectAnnotationsForPathContain(info, "test.proto", hpb_h, field_path, + { + {"sfield", Annotation::NONE}, + {"set_sfield", Annotation::SET}, + {"has_sfield", Annotation::NONE}, + {"clear_sfield", Annotation::NONE}, + }); + + // Check annotations for `rsfield`. + field_path = { + FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kFieldFieldNumber, + 1, + }; + ExpectAnnotationsForPathContain(info, "test.proto", hpb_h, field_path, + { + {"rsfield", Annotation::NONE}, + {"add_rsfield", Annotation::SET}, + {"mutable_rsfield", Annotation::ALIAS}, + {"rsfield_size", Annotation::NONE}, + }); +} + +} // namespace +} // namespace hpb +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/upb/bazel/private/upb_proto_library_internal/cc_library_func.bzl b/upb/bazel/private/upb_proto_library_internal/cc_library_func.bzl index 47fc5a1cc2..39cfb1e3f5 100644 --- a/upb/bazel/private/upb_proto_library_internal/cc_library_func.bzl +++ b/upb/bazel/private/upb_proto_library_internal/cc_library_func.bzl @@ -7,7 +7,7 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") def upb_use_cpp_toolchain(): return use_cpp_toolchain() -def cc_library_func(ctx, name, hdrs, srcs, copts, dep_ccinfos, includes = [], alwayslink = False): +def cc_library_func(ctx, name, hdrs, srcs, copts, dep_ccinfos, includes = [], alwayslink = False, **kwargs): """Like cc_library(), but callable from rules. Args: @@ -18,6 +18,8 @@ def cc_library_func(ctx, name, hdrs, srcs, copts, dep_ccinfos, includes = [], al copts: Additional options for cc compilation. dep_ccinfos: CcInfo providers of dependencies we should build/link against. includes: Additional include paths. + alwayslink: Whether the library should be always linked. + **kwargs: Additional arguments passed to cc_common.compile. Returns: CcInfo provider for this compilation. @@ -43,6 +45,7 @@ def cc_library_func(ctx, name, hdrs, srcs, copts, dep_ccinfos, includes = [], al public_hdrs = hdrs, user_compile_flags = copts, compilation_contexts = compilation_contexts, + **kwargs ) # buildifier: disable=unused-variable