mmo-ref/dep-protobuf
1
0
Fork 0
mirror of https://github.com/protocolbuffers/protobuf synced 2026-08-26 02:23:14 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #10254 from haberman/sync-stage

Browse source 
Integrate from Piper for C++, Java, and Python
This commit is contained in:
Joshua Haberman 2022-07-14 16:23:44 -07:00 • committed by GitHub
commit a972626551
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 325 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

25
CHANGES.txt
View file

@ -1,19 +1,18 @@
2022-07-01 Unreleased version
* Add TransferAllMessages() util function
* Break ZeroCopyOutputByteSink into its own file.
* Explicitly instantiate InternalMetadata members at UnknownFieldSet.
* Support kdoc for Kotlin
* Delete unused function: IsProto3Field.
C++
* Reduced .pb.o object file size slightly by explicitly instantiating
InternalMetadata templates in the runtime.
* Add C++20 keywords guarded by PROTOBUF_FUTURE_CPP20_KEYWORDS
* Escape Kotlin keywords in package names in proto generated code
* Fix StrAppend crashing on empty strings.
* Fixed crash in ThreadLocalStorage for pre-C++17 compilers on 32-bit ARM.
* Clarified that JSON API non-OK statuses are not a stable API.
2022-06-27 Unreleased version
* Handle reflection for message splitting.
* make metadata fields lazy.
* Extend visibility of plugin library to upb
* Modernize conformance_cpp.cc.
* Don't request 64-byte alignment unless the toolchain supports it.
Kotlin
* Kotlin generated code comments now use kdoc format instead of javadoc.
* Escape keywords in package names in proto generated code
Java
* Performance improvement for repeated use of FieldMaskUtil#merge by caching
constructed FieldMaskTrees.
2022-06-27 version 21.2 (C++/Java/Python/PHP/Objective-C/C#/Ruby)
C++

5
benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
View file

@ -12,9 +12,8 @@ import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
/**
* Basic benchmarks for Java protobuf parsing.
*/
/** Basic benchmarks for Java protobuf parsing. */
@SuppressWarnings("CheckReturnValue")
public class ProtoCaliperBenchmark {
public enum BenchmarkMessageType {
GOOGLE_MESSAGE1_PROTO3 {

5
java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java
View file

@ -30,14 +30,13 @@
package com.google.protobuf;
import junit.framework.TestCase;
import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar;
import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime;
import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo;
import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals;
import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof;
import junit.framework.TestCase;
/**
* Test generate equal and hash methods for the lite runtime.
*
@ -120,6 +119,6 @@ public class LiteEqualsAndHashTest extends TestCase {
public void testRecursiveHashcode() {
// This tests that we don't infinite loop.
TestRecursiveOneof.getDefaultInstance().hashCode();
int unused = TestRecursiveOneof.getDefaultInstance().hashCode();
}
}

16
python/google/protobuf/internal/descriptor_test.py
View file

@ -118,6 +118,22 @@ class DescriptorTest(unittest.TestCase):
def GetDescriptorPool(self):
return symbol_database.Default().pool
def testMissingPackage(self):
file_proto = descriptor_pb2.FileDescriptorProto(
name='some/filename/some.proto')
serialized = file_proto.SerializeToString()
pool = descriptor_pool.DescriptorPool()
file_descriptor = pool.AddSerializedFile(serialized)
self.assertEqual('', file_descriptor.package)
def testEmptyPackage(self):
file_proto = descriptor_pb2.FileDescriptorProto(
name='some/filename/some.proto', package='')
serialized = file_proto.SerializeToString()
pool = descriptor_pool.DescriptorPool()
file_descriptor = pool.AddSerializedFile(serialized)
self.assertEqual('', file_descriptor.package)
def testFindMethodByName(self):
service_descriptor = (unittest_custom_options_pb2.
TestServiceWithCustomOptions.DESCRIPTOR)

20
python/google/protobuf/internal/message_test.py
View file

@ -2455,6 +2455,26 @@ class Proto3Test(unittest.TestCase):
with self.assertRaises(ValueError):
unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801\ud801')
def testCrashNullAA(self):
self.assertEqual(
unittest_proto3_arena_pb2.TestAllTypes.NestedMessage(),
unittest_proto3_arena_pb2.TestAllTypes.NestedMessage())
def testCrashNullAB(self):
self.assertEqual(
unittest_proto3_arena_pb2.TestAllTypes.NestedMessage(),
unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message)
def testCrashNullBA(self):
self.assertEqual(
unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message,
unittest_proto3_arena_pb2.TestAllTypes.NestedMessage())
def testCrashNullBB(self):
self.assertEqual(
unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message,
unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message)

2
src/google/protobuf/arena.cc
View file

@ -382,7 +382,7 @@ void ThreadSafeArena::Init() {
void ThreadSafeArena::SetInitialBlock(void* mem, size_t size) {
SerialArena* serial = SerialArena::New({mem, size}, &thread_cache(),
arena_stats_.MutableStats());
serial->set_next(NULL);
serial->set_next(nullptr);
threads_.store(serial, std::memory_order_relaxed);
CacheSerialArena(serial);
}

34
src/google/protobuf/arena.h
View file

@ -120,7 +120,7 @@ struct ArenaOptions {
// here.
size_t max_block_size;
// An initial block of memory for the arena to use, or NULL for none. If
// An initial block of memory for the arena to use, or nullptr for none. If
// provided, the block must live at least as long as the arena itself. The
// creator of the Arena retains ownership of the block after the Arena is
// destroyed.
@ -143,7 +143,7 @@ struct ArenaOptions {
ArenaOptions()
: start_block_size(internal::AllocationPolicy::kDefaultStartBlockSize),
max_block_size(internal::AllocationPolicy::kDefaultMaxBlockSize),
initial_block(NULL),
initial_block(nullptr),
initial_block_size(0),
block_alloc(nullptr),
block_dealloc(nullptr),
@ -180,7 +180,7 @@ struct ArenaOptions {
#if PROTOBUF_RTTI
#define RTTI_TYPE_ID(type) (&typeid(type))
#else
#define RTTI_TYPE_ID(type) (NULL)
#define RTTI_TYPE_ID(type) (nullptr)
#endif
// Arena allocator. Arena allocation replaces ordinary (heap-based) allocation
@ -210,7 +210,7 @@ struct ArenaOptions {
// with `args` (without `arena`), called when a T is allocated on the heap;
// and a constructor callable with `Arena* arena, Args&&... args`, called when
// a T is allocated on an arena. If the second constructor is called with a
// NULL arena pointer, it must be equivalent to invoking the first
// null arena pointer, it must be equivalent to invoking the first
// (`args`-only) constructor.
//
// - The type T must have a particular type trait: a nested type
@ -220,7 +220,7 @@ struct ArenaOptions {
//
// - The type T *may* have the type trait |DestructorSkippable_|. If this type
// trait is present in the type, then its destructor will not be called if and
// only if it was passed a non-NULL arena pointer. If this type trait is not
// only if it was passed a non-null arena pointer. If this type trait is not
// present on the type, then its destructor is always called when the
// containing arena is destroyed.
//
@ -263,9 +263,9 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
void Init(const ArenaOptions&) {}
// API to create proto2 message objects on the arena. If the arena passed in
// is NULL, then a heap allocated object is returned. Type T must be a message
// defined in a .proto file with cc_enable_arenas set to true, otherwise a
// compilation error will occur.
// is nullptr, then a heap allocated object is returned. Type T must be a
// message defined in a .proto file with cc_enable_arenas set to true,
// otherwise a compilation error will occur.
//
// RepeatedField and RepeatedPtrField may also be instantiated directly on an
// arena with this method.
@ -342,7 +342,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
"CreateArray requires a trivially destructible type");
GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
<< "Requested size is too large to fit into size_t.";
if (arena == NULL) {
if (arena == nullptr) {
return static_cast<T*>(::operator new[](num_elements * sizeof(T)));
} else {
return arena->CreateInternalRawArray<T>(num_elements);
@ -386,7 +386,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// arena-allocated memory.
template <typename T>
PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* object) {
if (object != NULL) {
if (object != nullptr) {
impl_.AddCleanup(object, &internal::cleanup::arena_destruct_object<T>);
}
}
@ -401,9 +401,9 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
}
// Retrieves the arena associated with |value| if |value| is an arena-capable
// message, or NULL otherwise. If possible, the call resolves at compile time.
// Note that we can often devirtualize calls to `value->GetArena()` so usually
// calling this method is unnecessary.
// message, or nullptr otherwise. If possible, the call resolves at compile
// time. Note that we can often devirtualize calls to `value->GetArena()` so
// usually calling this method is unnecessary.
template <typename T>
PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) {
return GetArenaInternal(value);
@ -568,7 +568,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
static_assert(
InternalHelper<T>::is_arena_constructable::value,
"CreateMessage can only construct types that are ArenaConstructable");
if (arena == NULL) {
if (arena == nullptr) {
return new T(nullptr, static_cast<Args&&>(args)...);
} else {
return arena->DoCreateMessage<T>(static_cast<Args&&>(args)...);
@ -583,7 +583,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
static_assert(
InternalHelper<T>::is_arena_constructable::value,
"CreateMessage can only construct types that are ArenaConstructable");
if (arena == NULL) {
if (arena == nullptr) {
// Generated arena constructor T(Arena*) is protected. Call via
// InternalHelper.
return InternalHelper<T>::New();
@ -730,13 +730,13 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// using the virtual destructor instead.
template <typename T>
PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) {
if (object != NULL) {
if (object != nullptr) {
impl_.AddCleanup(object, &internal::arena_delete_object<MessageLite>);
}
}
template <typename T>
PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) {
if (object != NULL) {
if (object != nullptr) {
impl_.AddCleanup(object, &internal::arena_delete_object<T>);
}
}

54
src/google/protobuf/arena_unittest.cc
View file

@ -84,10 +84,10 @@ class Notifier {
class SimpleDataType {
public:
SimpleDataType() : notifier_(NULL) {}
SimpleDataType() : notifier_(nullptr) {}
void SetNotifier(Notifier* notifier) { notifier_ = notifier; }
virtual ~SimpleDataType() {
if (notifier_ != NULL) {
if (notifier_ != nullptr) {
notifier_->Notify();
}
};
@ -171,17 +171,17 @@ TEST(ArenaTest, DestructorSkippable) {
TEST(ArenaTest, BasicCreate) {
Arena arena;
EXPECT_TRUE(Arena::Create<int32_t>(&arena) != NULL);
EXPECT_TRUE(Arena::Create<int64_t>(&arena) != NULL);
EXPECT_TRUE(Arena::Create<float>(&arena) != NULL);
EXPECT_TRUE(Arena::Create<double>(&arena) != NULL);
EXPECT_TRUE(Arena::Create<std::string>(&arena) != NULL);
EXPECT_TRUE(Arena::Create<int32_t>(&arena) != nullptr);
EXPECT_TRUE(Arena::Create<int64_t>(&arena) != nullptr);
EXPECT_TRUE(Arena::Create<float>(&arena) != nullptr);
EXPECT_TRUE(Arena::Create<double>(&arena) != nullptr);
EXPECT_TRUE(Arena::Create<std::string>(&arena) != nullptr);
arena.Own(new int32_t);
arena.Own(new int64_t);
arena.Own(new float);
arena.Own(new double);
arena.Own(new std::string);
arena.Own<int>(NULL);
arena.Own<int>(nullptr);
Notifier notifier;
SimpleDataType* data = Arena::Create<SimpleDataType>(&arena);
data->SetNotifier(&notifier);
@ -196,7 +196,7 @@ TEST(ArenaTest, CreateAndConstCopy) {
Arena arena;
const std::string s("foo");
const std::string* s_copy = Arena::Create<std::string>(&arena, s);
EXPECT_TRUE(s_copy != NULL);
EXPECT_TRUE(s_copy != nullptr);
EXPECT_EQ("foo", s);
EXPECT_EQ("foo", *s_copy);
}
@ -205,7 +205,7 @@ TEST(ArenaTest, CreateAndNonConstCopy) {
Arena arena;
std::string s("foo");
const std::string* s_copy = Arena::Create<std::string>(&arena, s);
EXPECT_TRUE(s_copy != NULL);
EXPECT_TRUE(s_copy != nullptr);
EXPECT_EQ("foo", s);
EXPECT_EQ("foo", *s_copy);
}
@ -214,7 +214,7 @@ TEST(ArenaTest, CreateAndMove) {
Arena arena;
std::string s("foo");
const std::string* s_move = Arena::Create<std::string>(&arena, std::move(s));
EXPECT_TRUE(s_move != NULL);
EXPECT_TRUE(s_move != nullptr);
EXPECT_TRUE(s.empty()); // NOLINT
EXPECT_EQ("foo", *s_move);
}
@ -226,7 +226,7 @@ TEST(ArenaTest, CreateWithFourConstructorArguments) {
const MustBeConstructedWithOneThroughFour* new_object =
Arena::Create<MustBeConstructedWithOneThroughFour>(&arena, 1, "2", three,
&four);
EXPECT_TRUE(new_object != NULL);
EXPECT_TRUE(new_object != nullptr);
ASSERT_EQ(1, new_object->one_);
ASSERT_STREQ("2", new_object->two_);
ASSERT_EQ("3", new_object->three_);
@ -242,7 +242,7 @@ TEST(ArenaTest, CreateWithEightConstructorArguments) {
const MustBeConstructedWithOneThroughEight* new_object =
Arena::Create<MustBeConstructedWithOneThroughEight>(
&arena, 1, "2", three, &four, 5, "6", seven, eight);
EXPECT_TRUE(new_object != NULL);
EXPECT_TRUE(new_object != nullptr);
ASSERT_EQ(1, new_object->one_);
ASSERT_STREQ("2", new_object->two_);
ASSERT_EQ("3", new_object->three_);
@ -504,7 +504,7 @@ TEST(ArenaTest, ReleaseMessage) {
TestAllTypes::NestedMessage* released_null =
arena_message->release_optional_nested_message();
EXPECT_EQ(NULL, released_null);
EXPECT_EQ(nullptr, released_null);
}
TEST(ArenaTest, SetAllocatedString) {
@ -605,8 +605,8 @@ TEST(ArenaTest, SwapBetweenArenaAndNonArenaUsingReflection) {
}
TEST(ArenaTest, ReleaseFromArenaMessageMakesCopy) {
TestAllTypes::NestedMessage* nested_msg = NULL;
std::string* nested_string = NULL;
TestAllTypes::NestedMessage* nested_msg = nullptr;
std::string* nested_string = nullptr;
{
Arena arena;
TestAllTypes* arena_message = Arena::CreateMessage<TestAllTypes>(&arena);
@ -623,7 +623,7 @@ TEST(ArenaTest, ReleaseFromArenaMessageMakesCopy) {
#if PROTOBUF_RTTI
TEST(ArenaTest, ReleaseFromArenaMessageUsingReflectionMakesCopy) {
TestAllTypes::NestedMessage* nested_msg = NULL;
TestAllTypes::NestedMessage* nested_msg = nullptr;
// Note: no string: reflection API only supports releasing submessages.
{
Arena arena;
@ -1098,7 +1098,7 @@ TEST(ArenaTest, ArenaOneofReflection) {
EXPECT_TRUE(refl->HasOneof(*message, oneof));
submsg = refl->ReleaseMessage(message, msg_field);
EXPECT_FALSE(refl->HasOneof(*message, oneof));
EXPECT_TRUE(submsg->GetArena() == NULL);
EXPECT_TRUE(submsg->GetArena() == nullptr);
delete submsg;
}
@ -1111,7 +1111,7 @@ void TestSwapRepeatedField(Arena* arena1, Arena* arena2) {
TestAllTypes* t = Arena::CreateMessage<TestAllTypes>(arena1);
t->set_optional_string("field1");
t->set_optional_int32(i);
if (arena1 != NULL) {
if (arena1 != nullptr) {
field1.UnsafeArenaAddAllocated(t);
} else {
field1.AddAllocated(t);
@ -1121,7 +1121,7 @@ void TestSwapRepeatedField(Arena* arena1, Arena* arena2) {
TestAllTypes* t = Arena::CreateMessage<TestAllTypes>(arena2);
t->set_optional_string("field2");
t->set_optional_int32(i);
if (arena2 != NULL) {
if (arena2 != nullptr) {
field2.UnsafeArenaAddAllocated(t);
} else {
field2.AddAllocated(t);
@ -1154,12 +1154,12 @@ TEST(ArenaTest, SwapRepeatedFieldWithDifferentArenas) {
TEST(ArenaTest, SwapRepeatedFieldWithNoArenaOnRightHandSide) {
Arena arena;
TestSwapRepeatedField(&arena, NULL);
TestSwapRepeatedField(&arena, nullptr);
}
TEST(ArenaTest, SwapRepeatedFieldWithNoArenaOnLeftHandSide) {
Arena arena;
TestSwapRepeatedField(NULL, &arena);
TestSwapRepeatedField(nullptr, &arena);
}
TEST(ArenaTest, ExtensionsOnArena) {
@ -1218,11 +1218,11 @@ TEST(ArenaTest, RepeatedFieldOnArena) {
TestAllTypes* extracted_messages[5];
// ExtractSubrange should copy to the heap.
repeated_message.ExtractSubrange(0, 5, extracted_messages);
EXPECT_EQ(NULL, extracted_messages[0]->GetArena());
EXPECT_EQ(nullptr, extracted_messages[0]->GetArena());
// We need to free the heap-allocated messages to prevent a leak.
for (int i = 0; i < 5; i++) {
delete extracted_messages[i];
extracted_messages[i] = NULL;
extracted_messages[i] = nullptr;
}
}
@ -1342,7 +1342,7 @@ TEST(ArenaTest, MessageLiteOnArena) {
{
MessageLite* generic_message = prototype->New(&arena);
EXPECT_TRUE(generic_message != NULL);
EXPECT_TRUE(generic_message != nullptr);
EXPECT_EQ(&arena, generic_message->GetArena());
EXPECT_TRUE(generic_message->ParseFromString(serialized));
TestAllTypes* deserialized = static_cast<TestAllTypes*>(generic_message);
@ -1463,8 +1463,8 @@ TEST(ArenaTest, GetArenaShouldReturnTheArenaForArenaAllocatedMessages) {
TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaAllocatedMessages) {
ArenaMessage message;
const ArenaMessage* const_pointer_to_message = &message;
EXPECT_EQ(NULL, Arena::GetArena(&message));
EXPECT_EQ(NULL, Arena::GetArena(const_pointer_to_message));
EXPECT_EQ(nullptr, Arena::GetArena(&message));
EXPECT_EQ(nullptr, Arena::GetArena(const_pointer_to_message));
}
TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaCompatibleTypes) {

3
src/google/protobuf/compiler/command_line_interface.cc
View file

@ -1680,8 +1680,7 @@ CommandLineInterface::InterpretArgument(const std::string& name,
})) {
case google::protobuf::io::win32::ExpandWildcardsResult::kSuccess:
break;
case google::protobuf::io::win32::ExpandWildcardsResult::
kErrorNoMatchingFile:
case google::protobuf::io::win32::ExpandWildcardsResult::kErrorNoMatchingFile:
// Path does not exist, is not a file, or it's longer than MAX_PATH and
// long path handling is disabled.
std::cerr << "Invalid file name pattern or missing input file \""

7
src/google/protobuf/compiler/cpp/helpers.cc
View file

@ -925,6 +925,13 @@ bool HasLazyFields(const FileDescriptor* file, const Options& options,
bool ShouldSplit(const Descriptor*, const Options&) { return false; }
bool ShouldSplit(const FieldDescriptor*, const Options&) { return false; }
bool ShouldForceAllocationOnConstruction(const Descriptor* desc,
const Options& options) {
(void)desc;
(void)options;
return false;
}
static bool HasRepeatedFields(const Descriptor* descriptor) {
for (int i = 0; i < descriptor->field_count(); ++i) {
if (descriptor->field(i)->label() == FieldDescriptor::LABEL_REPEATED) {

5
src/google/protobuf/compiler/cpp/helpers.h
View file

@ -379,6 +379,11 @@ bool ShouldSplit(const Descriptor* desc, const Options& options);
// Is the given field being split out?
bool ShouldSplit(const FieldDescriptor* field, const Options& options);
// Should we generate code that force creating an allocation in the constructor
// of the given message?
bool ShouldForceAllocationOnConstruction(const Descriptor* desc,
const Options& options);
inline bool IsFieldUsed(const FieldDescriptor* /* field */,
const Options& /* options */) {
return true;

20
src/google/protobuf/compiler/cpp/message.cc
View file

@ -1269,9 +1269,9 @@ void MessageGenerator::GenerateFieldAccessorDefinitions(io::Printer* printer) {
GenerateSingularFieldHasBits(field, format);
}
if (!IsCrossFileMaybeMap(field)) {
GenerateFieldClear(field, true, format);
}
if (!IsCrossFileMaybeMap(field)) {
GenerateFieldClear(field, true, format);
}
// Generate type-specific accessors.
if (!IsFieldStripped(field, options_)) {
@ -2483,6 +2483,13 @@ void MessageGenerator::GenerateSharedConstructorCode(io::Printer* printer) {
field_generators_.get(field).GenerateConstructorCode(printer);
}
if (ShouldForceAllocationOnConstruction(descriptor_, options_)) {
format(
"#ifdef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n"
"$mutable_unknown_fields$;\n"
"#endif // PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n");
}
for (auto oneof : OneOfRange(descriptor_)) {
format("clear_has_$1$();\n", oneof->name());
}
@ -2722,6 +2729,13 @@ void MessageGenerator::GenerateCopyConstructorBody(io::Printer* printer) const {
" static_cast<size_t>(reinterpret_cast<char*>(&$last$) -\n"
" reinterpret_cast<char*>(&$first$)) + sizeof($last$));\n";
if (ShouldForceAllocationOnConstruction(descriptor_, options_)) {
format(
"#ifdef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n"
"$mutable_unknown_fields$;\n"
"#endif // PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n");
}
for (size_t i = 0; i < optimized_order_.size(); ++i) {
const FieldDescriptor* field = optimized_order_[i];
if (ShouldSplit(field, options_)) {

2
src/google/protobuf/compiler/subprocess.cc
View file

@ -46,8 +46,8 @@
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/message.h>
#include <google/protobuf/io/io_win32.h>
#include <google/protobuf/message.h>
namespace google {
namespace protobuf {

4
src/google/protobuf/io/coded_stream.h
View file

@ -680,7 +680,7 @@ class PROTOBUF_EXPORT EpsCopyOutputStream {
if (PROTOBUF_PREDICT_FALSE(end_ - ptr < size)) {
return WriteRawFallback(data, size, ptr);
}
std::memcpy(ptr, data, size);
std::memcpy(ptr, data, static_cast<unsigned int>(size));
return ptr + size;
}
// Writes the buffer specified by data, size to the stream. Possibly by
@ -1776,7 +1776,7 @@ inline void CodedOutputStream::WriteRawMaybeAliased(const void* data,
inline uint8_t* CodedOutputStream::WriteRawToArray(const void* data, int size,
uint8_t* target) {
memcpy(target, data, size);
memcpy(target, data, static_cast<unsigned int>(size));
return target + size;
}

54
src/google/protobuf/map.h
View file

@ -379,6 +379,7 @@ class Map {
public:
using key_type = Key;
using mapped_type = T;
using init_type = std::pair<Key, T>;
using value_type = MapPair<Key, T>;
using pointer = value_type*;
@ -421,6 +422,22 @@ class Map {
~Map() {}
private:
template <typename P>
struct SameAsElementReference
: std::is_same<typename std::remove_cv<
typename std::remove_reference<reference>::type>::type,
typename std::remove_cv<
typename std::remove_reference<P>::type>::type> {};
template <class P>
using RequiresInsertable =
typename std::enable_if<std::is_convertible<P, init_type>::value ||
SameAsElementReference<P>::value,
int>::type;
template <class P>
using RequiresNotInit =
typename std::enable_if<!std::is_same<P, init_type>::value, int>::type;
using Allocator = internal::MapAllocator<void*>;
// InnerMap is a generic hash-based map. It doesn't contain any
@ -1270,7 +1287,6 @@ class Map {
const_iterator cbegin() const { return begin(); }
const_iterator cend() const { return end(); }
// Capacity
size_type size() const { return elements_.size(); }
bool empty() const { return size() == 0; }
@ -1351,15 +1367,17 @@ class Map {
elements_.try_emplace(std::forward<K>(k), std::forward<Args>(args)...);
return std::pair<iterator, bool>(iterator(p.first), p.second);
}
std::pair<iterator, bool> insert(const value_type& value) {
return try_emplace(value.first, value.second);
std::pair<iterator, bool> insert(init_type&& value) {
return try_emplace(std::move(value.first), std::move(value.second));
}
std::pair<iterator, bool> insert(value_type&& value) {
return try_emplace(value.first, std::move(value.second));
template <typename P, RequiresInsertable<P> = 0>
std::pair<iterator, bool> insert(P&& value) {
return try_emplace(std::forward<P>(value).first,
std::forward<P>(value).second);
}
template <typename... Args>
std::pair<iterator, bool> emplace(Args&&... args) {
return insert(value_type(std::forward<Args>(args)...));
return EmplaceInternal(Rank0{}, std::forward<Args>(args)...);
}
template <class InputIt>
void insert(InputIt first, InputIt last) {
@ -1368,7 +1386,12 @@ class Map {
try_emplace(pair.first, pair.second);
}
}
void insert(std::initializer_list<value_type> values) {
void insert(std::initializer_list<init_type> values) {
insert(values.begin(), values.end());
}
template <typename P, RequiresNotInit<P> = 0,
RequiresInsertable<const P&> = 0>
void insert(std::initializer_list<P> values) {
insert(values.begin(), values.end());
}
@ -1429,6 +1452,23 @@ class Map {
}
private:
struct Rank1 {};
struct Rank0 : Rank1 {};
// We try to construct `init_type` from `Args` with a fall back to
// `value_type`. The latter is less desired as it unconditionally makes a copy
// of `value_type::first`.
template <typename... Args>
auto EmplaceInternal(Rank0, Args&&... args) ->
typename std::enable_if<std::is_constructible<init_type, Args...>::value,
std::pair<iterator, bool>>::type {
return insert(init_type(std::forward<Args>(args)...));
}
template <typename... Args>
std::pair<iterator, bool> EmplaceInternal(Rank1, Args&&... args) {
return insert(value_type(std::forward<Args>(args)...));
}
Arena* arena() const { return elements_.arena(); }
InnerMap elements_;

55
src/google/protobuf/map_test.inc
View file

@ -737,6 +737,47 @@ TEST_F(MapImplTest, InsertSingleRValue) {
EXPECT_FALSE(result2.second);
}
TEST_F(MapImplTest, InsertSingleBraceInitList) {
int32_t key = 0;
int32_t value1 = 100;
int32_t value2 = 101;
// Insert a non-existing key.
auto result1 = map_.insert({key, value1});
ExpectSingleElement(key, value1);
auto it1 = result1.first;
EXPECT_EQ(key, it1->first);
EXPECT_EQ(value1, it1->second);
EXPECT_TRUE(result1.second);
// Insert an existing key.
auto result2 = map_.insert({key, value2});
ExpectSingleElement(key, value1);
auto it2 = result2.first;
EXPECT_TRUE(it1 == it2);
EXPECT_FALSE(result2.second);
}
TEST_F(MapImplTest, InsertSingleBraceInitListTypeMismatch) {
int32_t key = 0;
int32_t value1 = 100;
int32_t value2 = 101;
Map<int64_t, int64_t> m;
// Insert a non-existing key.
auto result1 = m.insert({key, value1});
EXPECT_TRUE(result1.second);
// Insert an existing key.
auto result2 = m.insert({key, value2});
EXPECT_FALSE(result2.second);
EXPECT_TRUE(result1.first == result2.first);
}
TEST_F(MapImplTest, TryEmplace) {
using ::testing::Pair;
using ::testing::UnorderedElementsAre;
@ -769,6 +810,20 @@ TEST_F(MapImplTest, Emplace) {
m, UnorderedElementsAre(Pair(1, "one"), Pair(2, "two"), Pair(42, "aaa")));
}
TEST_F(MapImplTest, EmplaceKeyOnly) {
using ::testing::Pair;
using ::testing::UnorderedElementsAre;
Map<int32_t, std::string> m;
m.emplace(1);
EXPECT_EQ(m.size(), 1);
const int32_t key = 42;
m.emplace(key);
EXPECT_THAT(m, UnorderedElementsAre(Pair(1, ""), Pair(42, "")));
}
struct CountedInstance {
CountedInstance() { ++num_created; }
CountedInstance(const CountedInstance&) : CountedInstance() {}

5
src/google/protobuf/port_def.inc
View file

@ -191,6 +191,7 @@
#define PROTOBUF_FUTURE_REMOVE_CLEARED_API 1
// Used to escape C++20 keywords.
// TODO(mkruskal): ping b/238664698 when this lands in opensource.
// Owner: mkruskal@
#define PROTOBUF_FUTURE_CPP20_KEYWORDS 1
#else
@ -559,6 +560,10 @@
#error PROTOBUF_FORCE_COPY_DEFAULT_STRING was previously defined
#endif
#ifdef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION
#error PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION was previously defined
#endif
#ifdef PROTOBUF_FALLTHROUGH_INTENDED
#error PROTOBUF_FALLTHROUGH_INTENDED was previously defined
#endif

1
src/google/protobuf/port_undef.inc
View file

@ -75,6 +75,7 @@
#undef PROTOBUF_FORCE_RESET_IN_CLEAR
#undef PROTOBUF_FUZZ_MESSAGE_SPACE_USED_LONG
#undef PROTOBUF_FORCE_COPY_DEFAULT_STRING
#undef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION
#undef PROTOBUF_NAMESPACE_OPEN
#undef PROTOBUF_NAMESPACE_CLOSE
#undef PROTOBUF_UNUSED

6
src/google/protobuf/repeated_field.h
View file

@ -311,6 +311,7 @@ class RepeatedField final {
// This is public due to it being called by generated code.
inline void InternalSwap(RepeatedField* other);
private:
template <typename T> friend class Arena::InternalHelper;
@ -338,7 +339,7 @@ class RepeatedField final {
// current_size_. This function is intended to be the only place where
// current_size_ is modified.
inline int ExchangeCurrentSize(int new_size) {
int prev_size = current_size_;
const int prev_size = current_size_;
current_size_ = new_size;
return prev_size;
}
@ -355,6 +356,7 @@ class RepeatedField final {
}
};
// If total_size_ == 0 this points to an Arena otherwise it points to the
// elements member of a Rep struct. Using this invariant allows the storage of
// the arena pointer without an extra allocation in the constructor.
@ -471,7 +473,7 @@ class RepeatedField final {
void Add(Element val) {
if (index_ == capacity_) {
repeated_field_->ExchangeCurrentSize(index_);
repeated_field_->current_size_ = index_;
repeated_field_->Reserve(index_ + 1);
capacity_ = repeated_field_->total_size_;
buffer_ = repeated_field_->unsafe_elements();

12
src/google/protobuf/util/json_util.h
View file

@ -93,6 +93,9 @@ typedef JsonPrintOptions JsonOptions;
// Converts from protobuf message to JSON and appends it to |output|. This is a
// simple wrapper of BinaryToJsonString(). It will use the DescriptorPool of the
// passed-in message to resolve Any types.
//
// Please note that non-OK statuses are not a stable output of this API and
// subject to change without notice.
PROTOBUF_EXPORT util::Status MessageToJsonString(const Message& message,
std::string* output,
const JsonOptions& options);
@ -105,6 +108,9 @@ inline util::Status MessageToJsonString(const Message& message,
// Converts from JSON to protobuf message. This is a simple wrapper of
// JsonStringToBinary(). It will use the DescriptorPool of the passed-in
// message to resolve Any types.
//
// Please note that non-OK statuses are not a stable output of this API and
// subject to change without notice.
PROTOBUF_EXPORT util::Status JsonStringToMessage(
StringPiece input, Message* message, const JsonParseOptions& options);
@ -119,6 +125,9 @@ inline util::Status JsonStringToMessage(StringPiece input,
// 2. input is not valid protobuf wire format, or conflicts with the type
// information returned by TypeResolver.
// Note that unknown fields will be discarded silently.
//
// Please note that non-OK statuses are not a stable output of this API and
// subject to change without notice.
PROTOBUF_EXPORT util::Status BinaryToJsonStream(
TypeResolver* resolver, const std::string& type_url,
io::ZeroCopyInputStream* binary_input,
@ -150,6 +159,9 @@ inline util::Status BinaryToJsonString(TypeResolver* resolver,
// 1. TypeResolver fails to resolve a type.
// 2. input is not valid JSON format, or conflicts with the type
// information returned by TypeResolver.
//
// Please note that non-OK statuses are not a stable output of this API and
// subject to change without notice.
PROTOBUF_EXPORT util::Status JsonToBinaryStream(
TypeResolver* resolver, const std::string& type_url,
io::ZeroCopyInputStream* json_input,

140
src/google/protobuf/util/json_util_test.cc
View file

@ -74,6 +74,8 @@ using ::proto3::TestMessage;
using ::proto3::TestOneof;
using ::proto3::TestWrapper;
using ::proto_util_converter::testing::MapIn;
using ::testing::ElementsAre;
using ::testing::SizeIs;
// TODO(b/234474291): Use the gtest versions once that's available in OSS.
MATCHER_P(IsOkAndHolds, inner,
@ -301,9 +303,8 @@ TEST_P(JsonTest, TestAlwaysPrintEnumsAsInts) {
ASSERT_OK(parsed);
EXPECT_EQ(parsed->enum_value(), proto3::BAR);
EXPECT_EQ(parsed->repeated_enum_value_size(), 2);
EXPECT_EQ(parsed->repeated_enum_value(0), proto3::FOO);
EXPECT_EQ(parsed->repeated_enum_value(1), proto3::BAR);
EXPECT_THAT(parsed->repeated_enum_value(),
ElementsAre(proto3::FOO, proto3::BAR));
}
TEST_P(JsonTest, TestPrintEnumsAsIntsWithDefaultValue) {
@ -398,31 +399,14 @@ TEST_P(JsonTest, ParseMessage) {
EXPECT_EQ(m->enum_value(), proto3::EnumType::BAR);
EXPECT_EQ(m->message_value().value(), 2048);
ASSERT_EQ(m->repeated_bool_value_size(), 1);
EXPECT_TRUE(m->repeated_bool_value(0));
EXPECT_THAT(m->repeated_bool_value(), ElementsAre(true));
EXPECT_THAT(m->repeated_int32_value(), ElementsAre(0, -42));
EXPECT_THAT(m->repeated_uint64_value(), ElementsAre(1, 2));
EXPECT_THAT(m->repeated_double_value(), ElementsAre(1.5, -2));
EXPECT_THAT(m->repeated_string_value(), ElementsAre("foo", "bar ", ""));
EXPECT_THAT(m->repeated_enum_value(), ElementsAre(proto3::BAR, proto3::FOO));
ASSERT_EQ(m->repeated_int32_value_size(), 2);
EXPECT_EQ(m->repeated_int32_value(0), 0);
EXPECT_EQ(m->repeated_int32_value(1), -42);
ASSERT_EQ(m->repeated_uint64_value_size(), 2);
EXPECT_EQ(m->repeated_uint64_value(0), 1);
EXPECT_EQ(m->repeated_uint64_value(1), 2);
ASSERT_EQ(m->repeated_double_value_size(), 2);
EXPECT_EQ(m->repeated_double_value(0), 1.5);
EXPECT_EQ(m->repeated_double_value(1), -2);
ASSERT_EQ(m->repeated_string_value_size(), 3);
EXPECT_EQ(m->repeated_string_value(0), "foo");
EXPECT_EQ(m->repeated_string_value(1), "bar ");
EXPECT_EQ(m->repeated_string_value(2), "");
ASSERT_EQ(m->repeated_enum_value_size(), 2);
EXPECT_EQ(m->repeated_enum_value(0), proto3::EnumType::BAR);
EXPECT_EQ(m->repeated_enum_value(1), proto3::EnumType::FOO);
ASSERT_EQ(m->repeated_message_value_size(), 2);
ASSERT_THAT(m->repeated_message_value(), SizeIs(2));
EXPECT_EQ(m->repeated_message_value(0).value(), 40);
EXPECT_EQ(m->repeated_message_value(1).value(), 96);
@ -445,19 +429,9 @@ TEST_P(JsonTest, CurseOfAtob) {
}
)json");
ASSERT_OK(m);
EXPECT_EQ(m->repeated_bool_value_size(), 10);
EXPECT_FALSE(m->repeated_bool_value(0));
EXPECT_FALSE(m->repeated_bool_value(2));
EXPECT_FALSE(m->repeated_bool_value(4));
EXPECT_FALSE(m->repeated_bool_value(6));
EXPECT_FALSE(m->repeated_bool_value(8));
EXPECT_TRUE(m->repeated_bool_value(1));
EXPECT_TRUE(m->repeated_bool_value(3));
EXPECT_TRUE(m->repeated_bool_value(5));
EXPECT_TRUE(m->repeated_bool_value(7));
EXPECT_TRUE(m->repeated_bool_value(9));
EXPECT_THAT(m->repeated_bool_value(),
ElementsAre(false, true, false, true, false, true, false, true,
false, true));
}
TEST_P(JsonTest, ParseLegacySingleRepeatedField) {
@ -469,16 +443,11 @@ TEST_P(JsonTest, ParseLegacySingleRepeatedField) {
})json");
ASSERT_OK(m);
ASSERT_EQ(m->repeated_int32_value_size(), 1);
EXPECT_EQ(m->repeated_int32_value(0), 1997);
EXPECT_THAT(m->repeated_int32_value(), ElementsAre(1997));
EXPECT_THAT(m->repeated_string_value(), ElementsAre("oh no"));
EXPECT_THAT(m->repeated_enum_value(), ElementsAre(proto3::EnumType::BAR));
ASSERT_EQ(m->repeated_string_value_size(), 1);
EXPECT_EQ(m->repeated_string_value(0), "oh no");
ASSERT_EQ(m->repeated_enum_value_size(), 1);
EXPECT_EQ(m->repeated_enum_value(0), proto3::EnumType::BAR);
ASSERT_EQ(m->repeated_message_value_size(), 1);
ASSERT_THAT(m->repeated_message_value(), SizeIs(1));
EXPECT_EQ(m->repeated_message_value(0).value(), -1);
EXPECT_THAT(ToJson(*m),
@ -499,6 +468,13 @@ TEST_P(JsonTest, ParseMap) {
EXPECT_EQ(other->DebugString(), message.DebugString());
}
TEST_P(JsonTest, RepeatedMapKey) {
EXPECT_THAT(ToProto<TestMap>(R"json({
"twiceKey": 0,
"twiceKey": 1
})json"), StatusIs(util::StatusCode::kInvalidArgument));
}
TEST_P(JsonTest, ParsePrimitiveMapIn) {
MapIn message;
JsonPrintOptions print_options;
@ -535,10 +511,43 @@ TEST_P(JsonTest, ParseOverOneof) {
EXPECT_EQ(m.oneof_int32_value(), 5);
}
TEST_P(JsonTest, RepeatedSingularKeys) {
auto m = ToProto<TestMessage>(R"json({
"int32Value": 1,
"int32Value": 2
})json");
EXPECT_OK(m);
EXPECT_EQ(m->int32_value(), 2);
}
TEST_P(JsonTest, RepeatedRepeatedKeys) {
auto m = ToProto<TestMessage>(R"json({
"repeatedInt32Value": [1],
"repeatedInt32Value": [2, 3]
})json");
EXPECT_OK(m);
EXPECT_THAT(m->repeated_int32_value(), ElementsAre(1, 2, 3));
}
TEST_P(JsonTest, RepeatedOneofKeys) {
EXPECT_THAT(ToProto<TestOneof>(R"json({
"oneofInt32Value": 1,
"oneofStringValue": "foo"
})json"),
StatusIs(util::StatusCode::kInvalidArgument));
}
TEST_P(JsonTest, TestParseIgnoreUnknownFields) {
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_OK(ToProto<TestMessage>(R"({"unknownName":0})", options));
TestMessage m;
m.GetReflection()->MutableUnknownFields(&m)->AddFixed32(9001, 9001);
m.GetReflection()->MutableUnknownFields(&m)->AddFixed64(9001, 9001);
m.GetReflection()->MutableUnknownFields(&m)->AddVarint(9001, 9001);
m.GetReflection()->MutableUnknownFields(&m)->AddLengthDelimited(9001, "9001");
EXPECT_THAT(ToJson(m), IsOkAndHolds("{}"));
}
TEST_P(JsonTest, TestParseErrors) {
@ -578,9 +587,8 @@ TEST_P(JsonTest, TestDynamicMessage) {
EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString()));
EXPECT_EQ(generated.int32_value(), 1024);
ASSERT_EQ(generated.repeated_int32_value_size(), 2);
EXPECT_EQ(generated.repeated_int32_value(0), 1);
EXPECT_EQ(generated.repeated_int32_value(1), 2);
EXPECT_THAT(generated.repeated_int32_value(), ElementsAre(1, 2));
EXPECT_EQ(generated.message_value().value(), 2048);
ASSERT_EQ(generated.repeated_message_value_size(), 2);
EXPECT_EQ(generated.repeated_message_value(0).value(), 40);
@ -825,11 +833,11 @@ TEST_P(JsonTest, TestParsingEnumCaseSensitive) {
EXPECT_EQ(m.enum_value(), proto3::FOO);
}
TEST_P(JsonTest, TestParsingEnumLowercase) {
JsonParseOptions options;
options.case_insensitive_enum_parsing = true;
auto m = ToProto<TestMessage>(R"json({"enum_value": "TLSv1_2"})json", options);
auto m =
ToProto<TestMessage>(R"json({"enum_value": "TLSv1_2"})json", options);
ASSERT_OK(m);
EXPECT_THAT(m->enum_value(), proto3::TLSv1_2);
}
@ -851,10 +859,7 @@ TEST_P(JsonTest, TestOverwriteRepeated) {
m.add_repeated_int32_value(5);
ASSERT_OK(ToProto(m, R"json({"repeated_int32_value": [1, 2, 3]})json"));
EXPECT_EQ(m.repeated_int32_value_size(), 3);
EXPECT_EQ(m.repeated_int32_value(0), 1);
EXPECT_EQ(m.repeated_int32_value(1), 2);
EXPECT_EQ(m.repeated_int32_value(2), 3);
EXPECT_THAT(m.repeated_int32_value(), ElementsAre(1, 2, 3));
}
@ -869,7 +874,8 @@ TEST_P(JsonTest, TestDuration) {
EXPECT_EQ(m->value().seconds(), 123456);
EXPECT_EQ(m->value().nanos(), 789000000);
EXPECT_EQ(m->repeated_value().size(), 2);
EXPECT_THAT(m->repeated_value(), SizeIs(2));
EXPECT_EQ(m->repeated_value(0).seconds(), 0);
EXPECT_EQ(m->repeated_value(0).nanos(), 100000000);
EXPECT_EQ(m->repeated_value(1).seconds(), 999);
@ -904,7 +910,7 @@ TEST_P(JsonTest, TestTimestamp) {
EXPECT_EQ(m->value().seconds(), 825422400);
EXPECT_EQ(m->value().nanos(), 0);
EXPECT_EQ(m->repeated_value().size(), 1);
EXPECT_THAT(m->repeated_value(), SizeIs(1));
EXPECT_EQ(m->repeated_value(0).seconds(), 253402300799);
EXPECT_EQ(m->repeated_value(0).nanos(), 0);
@ -954,10 +960,7 @@ TEST_P(JsonTest, TestFieldMask) {
)json");
ASSERT_OK(m);
EXPECT_EQ(m->value().paths_size(), 2);
EXPECT_EQ(m->value().paths(0), "foo");
EXPECT_EQ(m->value().paths(1), "bar.baz_baz");
EXPECT_THAT(m->value().paths(), ElementsAre("foo", "bar.baz_baz"));
EXPECT_THAT(ToJson(*m), IsOkAndHolds(R"({"value":"foo,bar.bazBaz"})"));
auto m2 = ToProto<proto3::TestFieldMask>(R"json(
@ -969,8 +972,7 @@ TEST_P(JsonTest, TestFieldMask) {
)json");
ASSERT_OK(m2);
EXPECT_EQ(m2->value().paths_size(), 1);
EXPECT_EQ(m2->value().paths(0), "yep.really");
EXPECT_THAT(m2->value().paths(), ElementsAre("yep.really"));
}
TEST_P(JsonTest, TestLegalNullsInArray) {
@ -979,15 +981,15 @@ TEST_P(JsonTest, TestLegalNullsInArray) {
})json");
ASSERT_OK(m);
EXPECT_EQ(m->repeated_null_value_size(), 1);
EXPECT_EQ(m->repeated_null_value(0), google::protobuf::NULL_VALUE);
EXPECT_THAT(m->repeated_null_value(),
ElementsAre(google::protobuf::NULL_VALUE));
auto m2 = ToProto<proto3::TestValue>(R"json({
"repeatedValue": [null]
})json");
ASSERT_OK(m2);
EXPECT_EQ(m2->repeated_value_size(), 1);
ASSERT_THAT(m2->repeated_value(), SizeIs(1));
EXPECT_TRUE(m2->repeated_value(0).has_null_value());
}