mmo-ref/dep-protobuf
1
0
Fork 0
mirror of https://github.com/protocolbuffers/protobuf synced 2026-08-26 02:23:14 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Auto-format Py Proto python files.

Browse source 
PiperOrigin-RevId: 907755617
This commit is contained in:
Protobuf Team Bot 2026-04-29 13:39:31 -07:00 • committed by Copybara-Service
parent 1908079c5c
commit 6cd6ca4116
58 changed files with 5223 additions and 3333 deletions
Download patch file Download diff file Expand all files Collapse all files

4
python/dist/setup.py vendored
View file

@ -12,7 +12,7 @@ import glob
import os
import sys
from setuptools import setup, Extension, find_namespace_packages
from setuptools import Extension, find_namespace_packages, setup
def GetVersion():
@ -28,7 +28,7 @@ def GetVersion():
with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file:
file_globals = {}
exec(version_file.read(), file_globals) # pylint:disable=exec-used
return file_globals["__version__"]
return file_globals['__version__']
current_dir = os.path.dirname(os.path.abspath(__file__))

1
python/google/protobuf/any.py
View file

@ -14,7 +14,6 @@ from google.protobuf.message import Message
from google.protobuf.any_pb2 import Any
_MessageT = TypeVar('_MessageT', bound=Message)

2
python/google/protobuf/descriptor.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Descriptors essentially contain exactly the information found in a .proto
file, in types that make this information accessible in Python.
@ -101,7 +100,6 @@ _FEATURESET_ENUM_TYPE_CLOSED = 2
# users to notice and do not cause timeout.
_Deprecated.count = 100
_internal_create_key = object()

22
python/google/protobuf/descriptor_database.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Provides a container for DescriptorProtos."""
__author__ = 'matthewtoia@google.com (Matt Toia)'
@ -37,6 +36,7 @@ class DescriptorDatabase(object):
Args:
file_desc_proto: The FileDescriptorProto to add.
Raises:
DescriptorDatabaseConflictingDefinitionError: if an attempt is made to
add a proto with the same name but different definition than an
@ -47,7 +47,8 @@ class DescriptorDatabase(object):
self._file_desc_protos_by_file[proto_name] = file_desc_proto
elif self._file_desc_protos_by_file[proto_name] != file_desc_proto:
raise DescriptorDatabaseConflictingDefinitionError(
'%s already added, but with different descriptor.' % proto_name)
'%s already added, but with different descriptor.' % proto_name
)
else:
return
@ -143,7 +144,9 @@ class DescriptorDatabase(object):
raise KeyError(symbol)
def FindFileContainingExtension(
self, extendee_name: str, extension_number: int # pylint: disable=unused-argument
self,
extendee_name: str,
extension_number: int, # pylint: disable=unused-argument
) -> Optional['descriptor_pb2.FileDescriptorProto']:
# TODO: implement this API.
return None
@ -156,10 +159,15 @@ class DescriptorDatabase(object):
self, name: str, file_desc_proto: 'descriptor_pb2.FileDescriptorProto'
) -> None:
if name in self._file_desc_protos_by_symbol:
warn_msg = ('Conflict register for file "' + file_desc_proto.name +
'": ' + name +
' is already defined in file "' +
self._file_desc_protos_by_symbol[name].name + '"')
warn_msg = (
'Conflict register for file "'
+ file_desc_proto.name
+ '": '
+ name
+ ' is already defined in file "'
+ self._file_desc_protos_by_symbol[name].name
+ '"'
)
warnings.warn(warn_msg, RuntimeWarning)
self._file_desc_protos_by_symbol[name] = file_desc_proto

349
python/google/protobuf/descriptor_pool.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Provides DescriptorPool to use as a container for proto2 descriptors.
The DescriptorPool is used in conjection with a DescriptorDatabase to maintain
@ -71,12 +70,15 @@ def _OptionsOrNone(descriptor_proto):
def _IsMessageSetExtension(field):
return (field.is_extension and
field.containing_type.has_options and
field.containing_type.GetOptions().message_set_wire_format and
field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
not field.is_required and
not field.is_repeated)
return (
field.is_extension
and field.containing_type.has_options
and field.containing_type.GetOptions().message_set_wire_format
and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
and not field.is_required
and not field.is_repeated
)
_edition_defaults_lock = threading.Lock()
@ -86,9 +88,9 @@ class DescriptorPool(object):
if _USE_C_DESCRIPTORS:
def __new__(cls, descriptor_db=None):
# pylint: disable=protected-access
return descriptor._message.DescriptorPool(descriptor_db)
def __new__(cls, descriptor_db=None):
# pylint: disable=protected-access
return descriptor._message.DescriptorPool(descriptor_db)
def __init__(
self, descriptor_db=None, use_deprecated_legacy_json_field_conflicts=False
@ -139,7 +141,8 @@ class DescriptorPool(object):
(self._enum_descriptors, descriptor.EnumDescriptor),
(self._service_descriptors, descriptor.ServiceDescriptor),
(self._toplevel_extensions, descriptor.FieldDescriptor),
(self._top_enum_values, descriptor.EnumValueDescriptor)]:
(self._top_enum_values, descriptor.EnumValueDescriptor),
]:
if desc_name in register:
old_desc = register[desc_name]
if isinstance(old_desc, descriptor.EnumValueDescriptor):
@ -147,18 +150,24 @@ class DescriptorPool(object):
else:
old_file = old_desc.file.name
if not isinstance(desc, descriptor_type) or (
old_file != file_name):
error_msg = ('Conflict register for file "' + file_name +
'": ' + desc_name +
' is already defined in file "' +
old_file + '". Please fix the conflict by adding '
'package name on the proto file, or use different '
'name for the duplication.')
if not isinstance(desc, descriptor_type) or (old_file != file_name):
error_msg = (
'Conflict register for file "'
+ file_name
+ '": '
+ desc_name
+ ' is already defined in file "'
+ old_file
+ '". Please fix the conflict by adding '
'package name on the proto file, or use different '
'name for the duplication.'
)
if isinstance(desc, descriptor.EnumValueDescriptor):
error_msg += ('\nNote: enum values appear as '
'siblings of the enum type instead of '
'children of it.')
error_msg += (
'\nNote: enum values appear as '
'siblings of the enum type instead of '
'children of it.'
)
raise TypeError(error_msg)
@ -186,8 +195,10 @@ class DescriptorPool(object):
# pylint: disable=g-import-not-at-top
from google.protobuf import descriptor_pb2
file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString(
serialized_file_desc_proto)
serialized_file_desc_proto
)
file_desc = self._ConvertFileProtoToFileDescriptor(file_desc_proto)
file_desc.serialized_pb = serialized_file_desc_proto
return file_desc
@ -232,8 +243,10 @@ class DescriptorPool(object):
# Count the number of dots to see whether the enum is toplevel or nested
# in a message. We cannot use enum_desc.containing_type at this stage.
if enum_desc.file.package:
top_level = (enum_desc.full_name.count('.')
- enum_desc.file.package.count('.') == 1)
top_level = (
enum_desc.full_name.count('.') - enum_desc.file.package.count('.')
== 1
)
else:
top_level = enum_desc.full_name.count('.') == 0
if top_level:
@ -241,7 +254,8 @@ class DescriptorPool(object):
package = enum_desc.file.package
for enum_value in enum_desc.values:
full_name = _NormalizeFullyQualifiedName(
'.'.join((package, enum_value.name)))
'.'.join((package, enum_value.name))
)
self._CheckConflictRegister(enum_value, full_name, file_name)
self._top_enum_values[full_name] = enum_value
self._AddFileDescriptor(enum_desc.file)
@ -257,8 +271,9 @@ class DescriptorPool(object):
if not isinstance(service_desc, descriptor.ServiceDescriptor):
raise TypeError('Expected instance of descriptor.ServiceDescriptor.')
self._CheckConflictRegister(service_desc, service_desc.full_name,
service_desc.file.name)
self._CheckConflictRegister(
service_desc, service_desc.full_name, service_desc.file.name
)
self._service_descriptors[service_desc.full_name] = service_desc
# Never call this method. It is for internal usage only.
@ -274,41 +289,54 @@ class DescriptorPool(object):
TypeError: when the specified extension is not a
descriptor.FieldDescriptor.
"""
if not (isinstance(extension, descriptor.FieldDescriptor) and
extension.is_extension):
if not (
isinstance(extension, descriptor.FieldDescriptor)
and extension.is_extension
):
raise TypeError('Expected an extension descriptor.')
if extension.extension_scope is None:
self._CheckConflictRegister(
extension, extension.full_name, extension.file.name)
extension, extension.full_name, extension.file.name
)
self._toplevel_extensions[extension.full_name] = extension
try:
existing_desc = self._extensions_by_number[
extension.containing_type][extension.number]
existing_desc = self._extensions_by_number[extension.containing_type][
extension.number
]
except KeyError:
pass
else:
if extension is not existing_desc:
raise AssertionError(
'Extensions "%s" and "%s" both try to extend message type "%s" '
'with field number %d.' %
(extension.full_name, existing_desc.full_name,
extension.containing_type.full_name, extension.number))
'with field number %d.'
% (
extension.full_name,
existing_desc.full_name,
extension.containing_type.full_name,
extension.number,
)
)
self._extensions_by_number[extension.containing_type][
extension.number] = extension
extension.number
] = extension
self._extensions_by_name[extension.containing_type][
extension.full_name] = extension
extension.full_name
] = extension
# Also register MessageSet extensions with the type name.
if _IsMessageSetExtension(extension):
self._extensions_by_name[extension.containing_type][
extension.message_type.full_name] = extension
extension.message_type.full_name
] = extension
if hasattr(extension.containing_type, '_concrete_class'):
python_message._AttachFieldHelpers(
extension.containing_type._concrete_class, extension)
extension.containing_type._concrete_class, extension
)
# Never call this method. It is for internal usage only.
def _InternalAddFileDescriptor(self, file_desc):
@ -435,9 +463,11 @@ class DescriptorPool(object):
top_name, _, sub_name = symbol.rpartition('.')
try:
message = self.FindMessageTypeByName(top_name)
assert (sub_name in message.extensions_by_name or
sub_name in message.fields_by_name or
sub_name in message.enum_values_by_name)
assert (
sub_name in message.extensions_by_name
or sub_name in message.fields_by_name
or sub_name in message.enum_values_by_name
)
return message.file
except (KeyError, AssertionError):
raise KeyError('Cannot find a file containing %s' % symbol)
@ -579,7 +609,8 @@ class DescriptorPool(object):
"""
# Fallback to descriptor db if FindAllExtensionNumbers is provided.
if self._descriptor_db and hasattr(
self._descriptor_db, 'FindAllExtensionNumbers'):
self._descriptor_db, 'FindAllExtensionNumbers'
):
full_name = message_descriptor.full_name
try:
all_numbers = self._descriptor_db.FindAllExtensionNumbers(full_name)
@ -609,8 +640,7 @@ class DescriptorPool(object):
if not self._descriptor_db:
return
# Only supported when FindFileContainingExtension is provided.
if not hasattr(
self._descriptor_db, 'FindFileContainingExtension'):
if not hasattr(self._descriptor_db, 'FindFileContainingExtension'):
return
full_name = message_descriptor.full_name
@ -628,8 +658,10 @@ class DescriptorPool(object):
try:
self._ConvertFileProtoToFileDescriptor(file_proto)
except:
warn_msg = ('Unable to load proto file %s for extension number %d.' %
(file_proto.name, number))
warn_msg = 'Unable to load proto file %s for extension number %d.' % (
file_proto.name,
number,
)
warnings.warn(warn_msg, RuntimeWarning)
def FindServiceByName(self, full_name):
@ -839,33 +871,51 @@ class DescriptorPool(object):
# scope of available message types when defining the passed in
# file proto.
for dependency in built_deps:
scope.update(self._ExtractSymbols(
dependency.message_types_by_name.values()))
scope.update((_PrefixWithDot(enum.full_name), enum)
for enum in dependency.enum_types_by_name.values())
scope.update(
self._ExtractSymbols(dependency.message_types_by_name.values())
)
scope.update(
(_PrefixWithDot(enum.full_name), enum)
for enum in dependency.enum_types_by_name.values()
)
for message_type in file_proto.message_type:
message_desc = self._ConvertMessageDescriptor(
message_type, file_proto.package, file_descriptor, scope,
file_proto.syntax)
file_descriptor.message_types_by_name[message_desc.name] = (
message_desc)
message_type,
file_proto.package,
file_descriptor,
scope,
file_proto.syntax,
)
file_descriptor.message_types_by_name[message_desc.name] = message_desc
for enum_type in file_proto.enum_type:
file_descriptor.enum_types_by_name[enum_type.name] = (
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope, True))
self._ConvertEnumDescriptor(
enum_type,
file_proto.package,
file_descriptor,
None,
scope,
True,
)
)
for index, extension_proto in enumerate(file_proto.extension):
extension_desc = self._MakeFieldDescriptor(
extension_proto, file_proto.package, index, file_descriptor,
is_extension=True)
extension_proto,
file_proto.package,
index,
file_descriptor,
is_extension=True,
)
extension_desc.containing_type = self._GetTypeFromScope(
file_descriptor.package, extension_proto.extendee, scope)
self._SetFieldType(extension_proto, extension_desc,
file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = (
extension_desc)
file_descriptor.package, extension_proto.extendee, scope
)
self._SetFieldType(
extension_proto, extension_desc, file_descriptor.package, scope
)
file_descriptor.extensions_by_name[extension_desc.name] = extension_desc
for desc_proto in file_proto.message_type:
self._SetAllFieldTypes(file_proto.package, desc_proto, scope)
@ -876,14 +926,15 @@ class DescriptorPool(object):
desc_proto_prefix = ''
for desc_proto in file_proto.message_type:
desc = self._GetTypeFromScope(
desc_proto_prefix, desc_proto.name, scope)
desc = self._GetTypeFromScope(desc_proto_prefix, desc_proto.name, scope)
file_descriptor.message_types_by_name[desc_proto.name] = desc
for index, service_proto in enumerate(file_proto.service):
file_descriptor.services_by_name[service_proto.name] = (
self._MakeServiceDescriptor(service_proto, index, scope,
file_proto.package, file_descriptor))
self._MakeServiceDescriptor(
service_proto, index, scope, file_proto.package, file_descriptor
)
)
self._file_descriptors[file_proto.name] = file_descriptor
@ -902,8 +953,9 @@ class DescriptorPool(object):
return file_desc
def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None,
scope=None, syntax=None):
def _ConvertMessageDescriptor(
self, desc_proto, package=None, file_desc=None, scope=None, syntax=None
):
"""Adds the proto to the pool in the specified package.
Args:
@ -932,18 +984,26 @@ class DescriptorPool(object):
nested = [
self._ConvertMessageDescriptor(
nested, desc_name, file_desc, scope, syntax)
for nested in desc_proto.nested_type]
nested, desc_name, file_desc, scope, syntax
)
for nested in desc_proto.nested_type
]
enums = [
self._ConvertEnumDescriptor(enum, desc_name, file_desc, None,
scope, False)
for enum in desc_proto.enum_type]
fields = [self._MakeFieldDescriptor(field, desc_name, index, file_desc)
for index, field in enumerate(desc_proto.field)]
self._ConvertEnumDescriptor(
enum, desc_name, file_desc, None, scope, False
)
for enum in desc_proto.enum_type
]
fields = [
self._MakeFieldDescriptor(field, desc_name, index, file_desc)
for index, field in enumerate(desc_proto.field)
]
extensions = [
self._MakeFieldDescriptor(extension, desc_name, index, file_desc,
is_extension=True)
for index, extension in enumerate(desc_proto.extension)]
self._MakeFieldDescriptor(
extension, desc_name, index, file_desc, is_extension=True
)
for index, extension in enumerate(desc_proto.extension)
]
oneofs = [
# pylint: disable=g-complex-comprehension
descriptor.OneofDescriptor(
@ -954,7 +1014,8 @@ class DescriptorPool(object):
[],
_OptionsOrNone(desc),
# pylint: disable=protected-access
create_key=descriptor._internal_create_key)
create_key=descriptor._internal_create_key,
)
for index, desc in enumerate(desc_proto.oneof_decl)
]
extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range]
@ -997,8 +1058,15 @@ class DescriptorPool(object):
self._descriptors[desc_name] = desc
return desc
def _ConvertEnumDescriptor(self, enum_proto, package=None, file_desc=None,
containing_type=None, scope=None, top_level=False):
def _ConvertEnumDescriptor(
self,
enum_proto,
package=None,
file_desc=None,
containing_type=None,
scope=None,
top_level=False,
):
"""Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf.
Args:
@ -1007,8 +1075,8 @@ class DescriptorPool(object):
file_desc: The file containing the enum descriptor.
containing_type: The type containing this enum.
scope: Scope containing available types.
top_level: If True, the enum is a top level symbol. If False, the enum
is defined inside a message.
top_level: If True, the enum is a top level symbol. If False, the enum is
defined inside a message.
Returns:
The added descriptor
@ -1024,17 +1092,21 @@ class DescriptorPool(object):
else:
file_name = file_desc.name
values = [self._MakeEnumValueDescriptor(value, index)
for index, value in enumerate(enum_proto.value)]
desc = descriptor.EnumDescriptor(name=enum_proto.name,
full_name=enum_name,
filename=file_name,
file=file_desc,
values=values,
containing_type=containing_type,
options=_OptionsOrNone(enum_proto),
# pylint: disable=protected-access
create_key=descriptor._internal_create_key)
values = [
self._MakeEnumValueDescriptor(value, index)
for index, value in enumerate(enum_proto.value)
]
desc = descriptor.EnumDescriptor(
name=enum_proto.name,
full_name=enum_name,
filename=file_name,
file=file_desc,
values=values,
containing_type=containing_type,
options=_OptionsOrNone(enum_proto),
# pylint: disable=protected-access
create_key=descriptor._internal_create_key,
)
scope['.%s' % enum_name] = desc
self._CheckConflictRegister(desc, desc.full_name, desc.file.name)
self._enum_descriptors[enum_name] = desc
@ -1043,14 +1115,16 @@ class DescriptorPool(object):
if top_level:
for value in values:
full_name = _NormalizeFullyQualifiedName(
'.'.join((package, value.name)))
'.'.join((package, value.name))
)
self._CheckConflictRegister(value, full_name, file_name)
self._top_enum_values[full_name] = value
return desc
def _MakeFieldDescriptor(self, field_proto, message_name, index,
file_desc, is_extension=False):
def _MakeFieldDescriptor(
self, field_proto, message_name, index, file_desc, is_extension=False
):
"""Creates a field descriptor from a FieldDescriptorProto.
For message and enum type fields, this method will do a look up
@ -1099,7 +1173,8 @@ class DescriptorPool(object):
json_name=json_name,
file=file_desc,
# pylint: disable=protected-access
create_key=descriptor._internal_create_key)
create_key=descriptor._internal_create_key,
)
def _SetAllFieldTypes(self, package, desc_proto, scope):
"""Sets all the descriptor's fields's types.
@ -1124,10 +1199,12 @@ class DescriptorPool(object):
for field_proto, field_desc in zip(desc_proto.field, main_desc.fields):
self._SetFieldType(field_proto, field_desc, nested_package, scope)
for extension_proto, extension_desc in (
zip(desc_proto.extension, main_desc.extensions)):
for extension_proto, extension_desc in zip(
desc_proto.extension, main_desc.extensions
):
extension_desc.containing_type = self._GetTypeFromScope(
nested_package, extension_proto.extendee, scope)
nested_package, extension_proto.extendee, scope
)
self._SetFieldType(extension_proto, extension_desc, nested_package, scope)
for nested_type in desc_proto.nested_type:
@ -1154,10 +1231,13 @@ class DescriptorPool(object):
field_proto.type = descriptor.FieldDescriptor.TYPE_ENUM
field_desc.cpp_type = descriptor.FieldDescriptor.ProtoTypeToCppProtoType(
field_proto.type)
field_proto.type
)
if (field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE
or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP):
if (
field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE
or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP
):
field_desc.message_type = desc
if field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
@ -1168,8 +1248,10 @@ class DescriptorPool(object):
field_desc.default_value = []
elif field_proto.HasField('default_value'):
field_desc.has_default_value = True
if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or
field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT):
if (
field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE
or field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT
):
field_desc.default_value = float(field_proto.default_value)
elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING:
field_desc.default_value = field_proto.default_value
@ -1177,10 +1259,12 @@ class DescriptorPool(object):
field_desc.default_value = field_proto.default_value.lower() == 'true'
elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
field_desc.default_value = field_desc.enum_type.values_by_name[
field_proto.default_value].number
field_proto.default_value
].number
elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES:
field_desc.default_value = text_encoding.CUnescape(
field_proto.default_value)
field_proto.default_value
)
elif field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE:
field_desc.default_value = None
else:
@ -1188,11 +1272,13 @@ class DescriptorPool(object):
field_desc.default_value = int(field_proto.default_value)
else:
field_desc.has_default_value = False
if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or
field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT):
if (
field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE
or field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT
):
field_desc.default_value = 0.0
elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING:
field_desc.default_value = u''
field_desc.default_value = ''
elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL:
field_desc.default_value = False
elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
@ -1227,10 +1313,12 @@ class DescriptorPool(object):
options=_OptionsOrNone(value_proto),
type=None,
# pylint: disable=protected-access
create_key=descriptor._internal_create_key)
create_key=descriptor._internal_create_key,
)
def _MakeServiceDescriptor(self, service_proto, service_index, scope,
package, file_desc):
def _MakeServiceDescriptor(
self, service_proto, service_index, scope, package, file_desc
):
"""Make a protobuf ServiceDescriptor given a ServiceDescriptorProto.
Args:
@ -1249,9 +1337,12 @@ class DescriptorPool(object):
else:
service_name = service_proto.name
methods = [self._MakeMethodDescriptor(method_proto, service_name, package,
scope, index)
for index, method_proto in enumerate(service_proto.method)]
methods = [
self._MakeMethodDescriptor(
method_proto, service_name, package, scope, index
)
for index, method_proto in enumerate(service_proto.method)
]
desc = descriptor.ServiceDescriptor(
name=service_proto.name,
full_name=service_name,
@ -1260,13 +1351,15 @@ class DescriptorPool(object):
options=_OptionsOrNone(service_proto),
file=file_desc,
# pylint: disable=protected-access
create_key=descriptor._internal_create_key)
create_key=descriptor._internal_create_key,
)
self._CheckConflictRegister(desc, desc.full_name, desc.file.name)
self._service_descriptors[service_name] = desc
return desc
def _MakeMethodDescriptor(self, method_proto, service_name, package, scope,
index):
def _MakeMethodDescriptor(
self, method_proto, service_name, package, scope, index
):
"""Creates a method descriptor from a MethodDescriptorProto.
Args:
@ -1280,10 +1373,10 @@ class DescriptorPool(object):
An initialized MethodDescriptor object.
"""
full_name = '.'.join((service_name, method_proto.name))
input_type = self._GetTypeFromScope(
package, method_proto.input_type, scope)
input_type = self._GetTypeFromScope(package, method_proto.input_type, scope)
output_type = self._GetTypeFromScope(
package, method_proto.output_type, scope)
package, method_proto.output_type, scope
)
return descriptor.MethodDescriptor(
name=method_proto.name,
full_name=full_name,
@ -1295,13 +1388,15 @@ class DescriptorPool(object):
server_streaming=method_proto.server_streaming,
options=_OptionsOrNone(method_proto),
# pylint: disable=protected-access
create_key=descriptor._internal_create_key)
create_key=descriptor._internal_create_key,
)
def _ExtractSymbols(self, descriptors):
"""Pulls out all the symbols from descriptor protos.
Args:
descriptors: The messages to extract descriptors from.
Yields:
A two element tuple of the type name and descriptor object.
"""

1
python/google/protobuf/internal/__init__.py
View file

@ -4,4 +4,3 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd

39
python/google/protobuf/internal/api_implementation.py
View file

@ -4,9 +4,7 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Determine which implementation of the protobuf API is used in this process.
"""
"""Determine which implementation of the protobuf API is used in this process."""
import importlib
import os
@ -33,7 +31,8 @@ try:
# The compile-time constants in the _api_implementation module can be used to
# switch to a certain implementation of the Python API at build time.
_implementation_type = _ApiVersionToImplementationType(
_api_implementation.api_version)
_api_implementation.api_version
)
except ImportError:
pass # Unspecified by compiler flags.
@ -57,22 +56,26 @@ if _implementation_type is None:
else:
_implementation_type = 'python'
# This environment variable can be used to switch to a certain implementation
# of the Python API, overriding the compile-time constants in the
# _api_implementation module. Right now only 'python', 'cpp' and 'upb' are
# valid values. Any other value will raise error.
_implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION',
_implementation_type)
_implementation_type = os.getenv(
'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', _implementation_type
)
if _implementation_type not in ('python', 'cpp', 'upb'):
raise ValueError('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not '
'supported. Please set to \'python\', \'cpp\' or '
'\'upb\'.'.format(_implementation_type))
raise ValueError(
'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not '
"supported. Please set to 'python', 'cpp' or "
"'upb'.".format(_implementation_type)
)
if 'PyPy' in sys.version and _implementation_type == 'cpp':
warnings.warn('PyPy does not work yet with cpp protocol buffers. '
'Falling back to the python implementation.')
warnings.warn(
'PyPy does not work yet with cpp protocol buffers. '
'Falling back to the python implementation.'
)
_implementation_type = 'python'
_c_module = None
@ -81,24 +84,27 @@ if _implementation_type == 'cpp':
try:
# pylint: disable=g-import-not-at-top
from google.protobuf.pyext import _message
sys.modules['google3.net.proto2.python.internal.cpp._message'] = _message
_c_module = _message
del _message
except ImportError:
# TODO: fail back to python
warnings.warn(
'Selected implementation cpp is not available.')
warnings.warn('Selected implementation cpp is not available.')
pass
if _implementation_type == 'upb':
try:
# pylint: disable=g-import-not-at-top
from google._upb import _message
_c_module = _message
del _message
except ImportError:
warnings.warn('Selected implementation upb is not available. '
'Falling back to the python implementation.')
warnings.warn(
'Selected implementation upb is not available. '
'Falling back to the python implementation.'
)
_implementation_type = 'python'
pass
@ -118,6 +124,7 @@ try:
#
# pylint: disable=g-import-not-at-top,unused-import
from google.protobuf import enable_deterministic_proto_serialization
_python_deterministic_proto_serialization = True
except ImportError:
_python_deterministic_proto_serialization = False

27
python/google/protobuf/internal/builder.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Builds descriptors, message classes and services for generated _pb2.py.
This file is only called in python generated _pb2.py files. It builds
@ -14,11 +13,11 @@ in generated code.
__author__ = 'jieluo@google.com (Jie Luo)'
from google.protobuf.internal import enum_type_wrapper
from google.protobuf.internal import python_message
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import enum_type_wrapper
from google.protobuf.internal import python_message
_sym_db = _symbol_database.Default()
@ -39,7 +38,7 @@ def BuildMessageAndEnumDescriptors(file_des, module):
file_des: FileDescriptor of the .proto file
module: Generated _pb2 module
"""
for (name, msg_des) in file_des.message_types_by_name.items():
for name, msg_des in file_des.message_types_by_name.items():
module_name = '_' + name.upper()
module[module_name] = msg_des
_BuildNestedDescriptors(module, msg_des, module_name + '_')
@ -71,23 +70,23 @@ def BuildTopDescriptorsAndMessages(file_des, module_name, module):
"""
# top level enums
for (name, enum_des) in file_des.enum_types_by_name.items():
for name, enum_des in file_des.enum_types_by_name.items():
module['_' + name.upper()] = enum_des
module[name] = enum_type_wrapper.EnumTypeWrapper(enum_des)
for enum_value in enum_des.values:
module[enum_value.name] = enum_value.number
# top level extensions
for (name, extension_des) in file_des.extensions_by_name.items():
for name, extension_des in file_des.extensions_by_name.items():
module[name.upper() + '_FIELD_NUMBER'] = extension_des.number
module[name] = extension_des
# services
for (name, service) in file_des.services_by_name.items():
for name, service in file_des.services_by_name.items():
module['_' + name.upper()] = service
# Build messages.
for (name, msg_des) in file_des.message_types_by_name.items():
for name, msg_des in file_des.message_types_by_name.items():
module[name] = _BuildMessage(module_name, msg_des, '')
@ -112,11 +111,13 @@ def BuildServices(file_des, module_name, module):
# pylint: disable=g-import-not-at-top
from google.protobuf import service_reflection
# pylint: enable=g-import-not-at-top
for (name, service) in file_des.services_by_name.items():
for name, service in file_des.services_by_name.items():
module[name] = service_reflection.GeneratedServiceType(
name, (),
dict(DESCRIPTOR=service, __module__=module_name))
name, (), dict(DESCRIPTOR=service, __module__=module_name)
)
stub_name = name + '_Stub'
module[stub_name] = service_reflection.GeneratedServiceStubType(
stub_name, (module[name],),
dict(DESCRIPTOR=service, __module__=module_name))
stub_name,
(module[name],),
dict(DESCRIPTOR=service, __module__=module_name),
)

142
python/google/protobuf/internal/containers.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains container classes to represent different protocol buffer types.
This file defines container classes which represent categories of protocol
@ -35,13 +34,13 @@ from typing import (
overload,
)
_T = TypeVar('_T')
_K = TypeVar('_K')
_V = TypeVar('_V')
from google.protobuf.descriptor import FieldDescriptor
class BaseContainer(Sequence[_T]):
"""Base container class."""
@ -49,11 +48,11 @@ class BaseContainer(Sequence[_T]):
__slots__ = ['_message_listener', '_values']
def __init__(self, message_listener: Any) -> None:
"""
Args:
message_listener: A MessageListener implementation.
The RepeatedScalarFieldContainer will call this object's
Modified() method when it is modified.
"""Args:
message_listener: A MessageListener implementation.
The RepeatedScalarFieldContainer will call this object's
Modified() method when it is modified.
"""
self._message_listener = message_listener
self._values = []
@ -115,11 +114,11 @@ class RepeatedScalarFieldContainer(BaseContainer[_T], MutableSequence[_T]):
) -> None:
"""Args:
message_listener: A MessageListener implementation. The
RepeatedScalarFieldContainer will call this object's Modified() method
when it is modified.
type_checker: A type_checkers.ValueChecker instance to run on elements
inserted into this container.
message_listener: A MessageListener implementation. The
RepeatedScalarFieldContainer will call this object's Modified() method
when it is modified.
type_checker: A type_checkers.ValueChecker instance to run on elements
inserted into this container.
"""
super().__init__(message_listener)
self._type_checker = type_checker
@ -150,6 +149,7 @@ class RepeatedScalarFieldContainer(BaseContainer[_T], MutableSequence[_T]):
other: Union['RepeatedScalarFieldContainer[_T]', Iterable[_T]],
) -> None:
"""Appends the contents of another repeated field of the same type to this
one. We do not check the types of the individual fields.
"""
self._values.extend(other)
@ -212,7 +212,8 @@ class RepeatedScalarFieldContainer(BaseContainer[_T], MutableSequence[_T]):
def __reduce__(self, **kwargs) -> NoReturn:
raise pickle.PickleError(
"Can't pickle repeated scalar fields, convert to list first")
"Can't pickle repeated scalar fields, convert to list first"
)
def __array__(self, dtype=None, copy=None):
import numpy as np
@ -255,26 +256,27 @@ class RepeatedCompositeFieldContainer(BaseContainer[_T], MutableSequence[_T]):
__slots__ = ['_message_descriptor']
def __init__(self, message_listener: Any, message_descriptor: Any) -> None:
"""
Note that we pass in a descriptor instead of the generated directly,
"""Note that we pass in a descriptor instead of the generated directly,
since at the time we construct a _RepeatedCompositeFieldContainer we
haven't yet necessarily initialized the type that will be contained in the
container.
Args:
message_listener: A MessageListener implementation.
The RepeatedCompositeFieldContainer will call this object's
Modified() method when it is modified.
message_listener: A MessageListener implementation. The
RepeatedCompositeFieldContainer will call this object's Modified()
method when it is modified.
message_descriptor: A Descriptor instance describing the protocol type
that should be present in this container. We'll use the
_concrete_class field of this descriptor when the client calls add().
that should be present in this container. We'll use the _concrete_class
field of this descriptor when the client calls add().
"""
super().__init__(message_listener)
self._message_descriptor = message_descriptor
def add(self, **kwargs: Any) -> _T:
"""Adds a new element at the end of the list and returns it. Keyword
arguments may be used to initialize the element.
"""Adds a new element at the end of the list and returns it.
Keyword arguments may be used to initialize the element.
"""
new_element = self._message_descriptor._concrete_class(**kwargs)
new_element._SetListener(self._message_listener)
@ -321,6 +323,7 @@ class RepeatedCompositeFieldContainer(BaseContainer[_T], MutableSequence[_T]):
other: Union['RepeatedCompositeFieldContainer[_T]', Iterable[_T]],
) -> None:
"""Appends the contents of another repeated field of the same type to this
one, copying each individual message.
"""
self.extend(other)
@ -349,7 +352,8 @@ class RepeatedCompositeFieldContainer(BaseContainer[_T], MutableSequence[_T]):
# structurally compatible with typing.MutableSequence. It is
# otherwise unsupported and will always raise an error.
raise TypeError(
f'{self.__class__.__name__} object does not support item assignment')
f'{self.__class__.__name__} object does not support item assignment'
)
def __delitem__(self, key: Union[int, slice]) -> None:
"""Deletes the item at the specified position."""
@ -361,8 +365,10 @@ class RepeatedCompositeFieldContainer(BaseContainer[_T], MutableSequence[_T]):
if self is other:
return True
if not isinstance(other, self.__class__):
raise TypeError('Can only compare repeated composite fields against '
'other repeated composite fields.')
raise TypeError(
'Can only compare repeated composite fields against '
'other repeated composite fields.'
)
return self._values == other._values
@ -370,8 +376,13 @@ class ScalarMap(MutableMapping[_K, _V]):
"""Simple, type-checked, dict-like container for holding repeated scalars."""
# Disallows assignment to other attributes.
__slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener',
'_entry_descriptor']
__slots__ = [
'_key_checker',
'_value_checker',
'_values',
'_message_listener',
'_entry_descriptor',
]
def __init__(
self,
@ -380,16 +391,16 @@ class ScalarMap(MutableMapping[_K, _V]):
value_checker: Any,
entry_descriptor: Any,
) -> None:
"""
Args:
message_listener: A MessageListener implementation.
The ScalarMap will call this object's Modified() method when it
is modified.
key_checker: A type_checkers.ValueChecker instance to run on keys
inserted into this container.
value_checker: A type_checkers.ValueChecker instance to run on values
inserted into this container.
entry_descriptor: The MessageDescriptor of a map entry: key and value.
"""Args:
message_listener: A MessageListener implementation.
The ScalarMap will call this object's Modified() method when it
is modified.
key_checker: A type_checkers.ValueChecker instance to run on keys
inserted into this container.
value_checker: A type_checkers.ValueChecker instance to run on values
inserted into this container.
entry_descriptor: The MessageDescriptor of a map entry: key and value.
"""
self._message_listener = message_listener
self._key_checker = key_checker
@ -479,8 +490,13 @@ class MessageMap(MutableMapping[_K, _V]):
"""Simple, type-checked, dict-like container for with submessage values."""
# Disallows assignment to other attributes.
__slots__ = ['_key_checker', '_values', '_message_listener',
'_message_descriptor', '_entry_descriptor']
__slots__ = [
'_key_checker',
'_values',
'_message_listener',
'_message_descriptor',
'_entry_descriptor',
]
def __init__(
self,
@ -489,16 +505,16 @@ class MessageMap(MutableMapping[_K, _V]):
key_checker: Any,
entry_descriptor: Any,
) -> None:
"""
Args:
message_listener: A MessageListener implementation.
The ScalarMap will call this object's Modified() method when it
is modified.
key_checker: A type_checkers.ValueChecker instance to run on keys
inserted into this container.
value_checker: A type_checkers.ValueChecker instance to run on values
inserted into this container.
entry_descriptor: The MessageDescriptor of a map entry: key and value.
"""Args:
message_listener: A MessageListener implementation.
The ScalarMap will call this object's Modified() method when it
is modified.
key_checker: A type_checkers.ValueChecker instance to run on keys
inserted into this container.
value_checker: A type_checkers.ValueChecker instance to run on values
inserted into this container.
entry_descriptor: The MessageDescriptor of a map entry: key and value.
"""
self._message_listener = message_listener
self._message_descriptor = message_descriptor
@ -622,9 +638,11 @@ class _UnknownField:
if self is other:
return True
# pylint: disable=protected-access
return (self._field_number == other._field_number and
self._wire_type == other._wire_type and
self._data == other._data)
return (
self._field_number == other._field_number
and self._wire_type == other._wire_type
and self._data == other._data
)
class UnknownFieldRef: # pylint: disable=missing-class-docstring
@ -635,11 +653,13 @@ class UnknownFieldRef: # pylint: disable=missing-class-docstring
def _check_valid(self):
if not self._parent:
raise ValueError('UnknownField does not exist. '
'The parent message might be cleared.')
raise ValueError(
'UnknownField does not exist. The parent message might be cleared.'
)
if self._index >= len(self._parent):
raise ValueError('UnknownField does not exist. '
'The parent message might be cleared.')
raise ValueError(
'UnknownField does not exist. The parent message might be cleared.'
)
@property
def field_number(self):
@ -671,8 +691,9 @@ class UnknownFieldSet:
def __getitem__(self, index):
if self._values is None:
raise ValueError('UnknownFields does not exist. '
'The parent message might be cleared.')
raise ValueError(
'UnknownFields does not exist. The parent message might be cleared.'
)
size = len(self._values)
if index < 0:
index += size
@ -686,8 +707,9 @@ class UnknownFieldSet:
def __len__(self):
if self._values is None:
raise ValueError('UnknownFields does not exist. '
'The parent message might be cleared.')
raise ValueError(
'UnknownFields does not exist. The parent message might be cleared.'
)
return len(self._values)
def _add(self, field_number, wire_type, data):

215
python/google/protobuf/internal/decoder.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Code for decoding protocol buffer primitives.
This code is very similar to encoder.py -- read the docs for that module first.
@ -66,7 +65,6 @@ from google.protobuf.internal import containers
from google.protobuf.internal import encoder
from google.protobuf.internal import wire_format
# This is not for optimization, but rather to avoid conflicts with local
# variables named "message".
_DecodeError = message.DecodeError
@ -103,7 +101,7 @@ def _VarintDecoder(mask, result_type):
decoder returns a (value, new_pos) pair.
"""
def DecodeVarint(buffer, pos: int=None):
def DecodeVarint(buffer, pos: int = None):
result = 0
shift = 0
while 1:
@ -120,7 +118,7 @@ def _VarintDecoder(mask, result_type):
else:
b = buffer[pos]
pos += 1
result |= ((b & 0x7f) << shift)
result |= (b & 0x7F) << shift
if not (b & 0x80):
result &= mask
result = result_type(result)
@ -143,7 +141,7 @@ def _SignedVarintDecoder(bits, result_type):
shift = 0
while 1:
b = buffer[pos]
result |= ((b & 0x7f) << shift)
result |= (b & 0x7F) << shift
pos += 1
if not (b & 0x80):
result &= mask
@ -153,8 +151,10 @@ def _SignedVarintDecoder(bits, result_type):
shift += 7
if shift >= 64:
raise _DecodeError('Too many bytes when decoding varint.')
return DecodeVarint
# All 32-bit and 64-bit values are represented as int.
_DecodeVarint = _VarintDecoder((1 << 64) - 1, int)
_DecodeSignedVarint = _SignedVarintDecoder(64, int)
@ -199,7 +199,7 @@ def DecodeTag(tag_bytes):
Returns:
Tuple[int, int] of the tag field number and wire type.
"""
(tag, _) = _DecodeVarint(tag_bytes, 0)
tag, _ = _DecodeVarint(tag_bytes, 0)
return wire_format.UnpackTag(tag)
@ -215,10 +215,17 @@ def _SimpleDecoder(wire_type, decode_value):
_DecodeVarint()
"""
def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default,
clear_if_default=False):
def SpecificDecoder(
field_number,
is_repeated,
is_packed,
key,
new_default,
clear_if_default=False,
):
if is_packed:
local_DecodeVarint = _DecodeVarint
def DecodePackedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -226,15 +233,15 @@ def _SimpleDecoder(wire_type, decode_value):
value = field_dict.get(key)
if value is None:
value = field_dict.setdefault(key, new_default(message))
(endpoint, pos) = local_DecodeVarint(buffer, pos)
endpoint, pos = local_DecodeVarint(buffer, pos)
endpoint += pos
if endpoint > end:
raise _DecodeError('Truncated message.')
while pos < endpoint:
(element, pos) = decode_value(buffer, pos)
element, pos = decode_value(buffer, pos)
value.append(element)
if pos > endpoint:
del value[-1] # Discard corrupt value.
del value[-1] # Discard corrupt value.
raise _DecodeError('Packed element was truncated.')
return pos
@ -242,6 +249,7 @@ def _SimpleDecoder(wire_type, decode_value):
elif is_repeated:
tag_bytes = encoder.TagBytes(field_number, wire_type)
tag_len = len(tag_bytes)
def DecodeRepeatedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -250,7 +258,7 @@ def _SimpleDecoder(wire_type, decode_value):
if value is None:
value = field_dict.setdefault(key, new_default(message))
while 1:
(element, new_pos) = decode_value(buffer, pos)
element, new_pos = decode_value(buffer, pos)
value.append(element)
# Predict that the next tag is another copy of the same repeated
# field.
@ -266,7 +274,7 @@ def _SimpleDecoder(wire_type, decode_value):
def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
del current_depth # unused
(new_value, pos) = decode_value(buffer, pos)
new_value, pos = decode_value(buffer, pos)
if pos > end:
raise _DecodeError('Truncated message.')
if clear_if_default and IsDefaultScalarValue(new_value):
@ -282,6 +290,7 @@ def _SimpleDecoder(wire_type, decode_value):
def _ModifiedDecoder(wire_type, decode_value, modify_value):
"""Like SimpleDecoder but additionally invokes modify_value on every value
before storing it. Usually modify_value is ZigZagDecode.
"""
@ -289,8 +298,9 @@ def _ModifiedDecoder(wire_type, decode_value, modify_value):
# not enough to make a significant difference.
def InnerDecode(buffer, pos):
(result, new_pos) = decode_value(buffer, pos)
result, new_pos = decode_value(buffer, pos)
return (modify_value(result), new_pos)
return _SimpleDecoder(wire_type, InnerDecode)
@ -316,6 +326,7 @@ def _StructPackDecoder(wire_type, format):
new_pos = pos + value_size
result = local_unpack(format, buffer[pos:new_pos])[0]
return (result, new_pos)
return _SimpleDecoder(wire_type, InnerDecode)
@ -347,7 +358,7 @@ def _FloatDecoder():
# If this value has all its exponent bits set, then it's non-finite.
# In Python 2.4, struct.unpack will convert it to a finite 64-bit value.
# To avoid that, we parse it specially.
if (float_bytes[3:4] in b'\x7F\xFF' and float_bytes[2:3] >= b'\x80'):
if float_bytes[3:4] in b'\x7F\xFF' and float_bytes[2:3] >= b'\x80':
# If at least one significand bit is set...
if float_bytes[0:3] != b'\x00\x00\x80':
return (math.nan, new_pos)
@ -361,6 +372,7 @@ def _FloatDecoder():
# handling blocks every time we parse one value.
result = local_unpack('<f', float_bytes)[0]
return (result, new_pos)
return _SimpleDecoder(wire_format.WIRETYPE_FIXED32, InnerDecode)
@ -391,9 +403,11 @@ def _DoubleDecoder():
# If this value has all its exponent bits set and at least one significand
# bit set, it's not a number. In Python 2.4, struct.unpack will treat it
# as inf or -inf. To avoid that, we treat it specially.
if ((double_bytes[7:8] in b'\x7F\xFF')
if (
(double_bytes[7:8] in b'\x7F\xFF')
and (double_bytes[6:7] >= b'\xF0')
and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')):
and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')
):
return (math.nan, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert
@ -401,15 +415,23 @@ def _DoubleDecoder():
# handling blocks every time we parse one value.
result = local_unpack('<d', double_bytes)[0]
return (result, new_pos)
return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode)
def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
clear_if_default=False):
def EnumDecoder(
field_number,
is_repeated,
is_packed,
key,
new_default,
clear_if_default=False,
):
"""Returns a decoder for enum field."""
enum_type = key.enum_type
if is_packed:
local_DecodeVarint = _DecodeVarint
def DecodePackedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -429,28 +451,30 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
value = field_dict.get(key)
if value is None:
value = field_dict.setdefault(key, new_default(message))
(endpoint, pos) = local_DecodeVarint(buffer, pos)
endpoint, pos = local_DecodeVarint(buffer, pos)
endpoint += pos
if endpoint > end:
raise _DecodeError('Truncated message.')
while pos < endpoint:
value_start_pos = pos
(element, pos) = _DecodeSignedVarint32(buffer, pos)
element, pos = _DecodeSignedVarint32(buffer, pos)
# pylint: disable=protected-access
if element in enum_type.values_by_number:
value.append(element)
else:
if not message._unknown_fields:
message._unknown_fields = []
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_VARINT)
tag_bytes = encoder.TagBytes(
field_number, wire_format.WIRETYPE_VARINT
)
message._unknown_fields.append(
(tag_bytes, buffer[value_start_pos:pos].tobytes()))
(tag_bytes, buffer[value_start_pos:pos].tobytes())
)
# pylint: enable=protected-access
if pos > endpoint:
if element in enum_type.values_by_number:
del value[-1] # Discard corrupt value.
del value[-1] # Discard corrupt value.
else:
del message._unknown_fields[-1]
# pylint: enable=protected-access
@ -461,6 +485,7 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
elif is_repeated:
tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT)
tag_len = len(tag_bytes)
def DecodeRepeatedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -481,7 +506,7 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
if value is None:
value = field_dict.setdefault(key, new_default(message))
while 1:
(element, new_pos) = _DecodeSignedVarint32(buffer, pos)
element, new_pos = _DecodeSignedVarint32(buffer, pos)
# pylint: disable=protected-access
if element in enum_type.values_by_number:
value.append(element)
@ -489,7 +514,8 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
if not message._unknown_fields:
message._unknown_fields = []
message._unknown_fields.append(
(tag_bytes, buffer[pos:new_pos].tobytes()))
(tag_bytes, buffer[pos:new_pos].tobytes())
)
# pylint: enable=protected-access
# Predict that the next tag is another copy of the same repeated
# field.
@ -518,7 +544,7 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
"""
del current_depth # unused
value_start_pos = pos
(enum_value, pos) = _DecodeSignedVarint32(buffer, pos)
enum_value, pos = _DecodeSignedVarint32(buffer, pos)
if pos > end:
raise _DecodeError('Truncated message.')
if clear_if_default and IsDefaultScalarValue(enum_value):
@ -530,10 +556,10 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
else:
if not message._unknown_fields:
message._unknown_fields = []
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_VARINT)
tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT)
message._unknown_fields.append(
(tag_bytes, buffer[value_start_pos:pos].tobytes()))
(tag_bytes, buffer[value_start_pos:pos].tobytes())
)
# pylint: enable=protected-access
return pos
@ -542,38 +568,44 @@ def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
# --------------------------------------------------------------------
Int32Decoder = _SimpleDecoder(
wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32)
wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32
)
Int64Decoder = _SimpleDecoder(
wire_format.WIRETYPE_VARINT, _DecodeSignedVarint)
Int64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeSignedVarint)
UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32)
UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint)
SInt32Decoder = _ModifiedDecoder(
wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode)
wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode
)
SInt64Decoder = _ModifiedDecoder(
wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode)
wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode
)
# Note that Python conveniently guarantees that when using the '<' prefix on
# formats, they will also have the same size across all platforms (as opposed
# to without the prefix, where their sizes depend on the C compiler's basic
# type sizes).
Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<I')
Fixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<Q')
Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<I')
Fixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<Q')
SFixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<i')
SFixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<q')
FloatDecoder = _FloatDecoder()
DoubleDecoder = _DoubleDecoder()
BoolDecoder = _ModifiedDecoder(
wire_format.WIRETYPE_VARINT, _DecodeVarint, bool)
BoolDecoder = _ModifiedDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint, bool)
def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
clear_if_default=False):
def StringDecoder(
field_number,
is_repeated,
is_packed,
key,
new_default,
clear_if_default=False,
):
"""Returns a decoder for a string field."""
local_DecodeVarint = _DecodeVarint
@ -592,9 +624,11 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
assert not is_packed
if is_repeated:
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_LENGTH_DELIMITED)
tag_bytes = encoder.TagBytes(
field_number, wire_format.WIRETYPE_LENGTH_DELIMITED
)
tag_len = len(tag_bytes)
def DecodeRepeatedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -603,7 +637,7 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
if value is None:
value = field_dict.setdefault(key, new_default(message))
while 1:
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated string.')
@ -619,7 +653,7 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
del current_depth # unused
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated string.')
@ -632,17 +666,25 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
return DecodeField
def BytesDecoder(field_number, is_repeated, is_packed, key, new_default,
clear_if_default=False):
def BytesDecoder(
field_number,
is_repeated,
is_packed,
key,
new_default,
clear_if_default=False,
):
"""Returns a decoder for a bytes field."""
local_DecodeVarint = _DecodeVarint
assert not is_packed
if is_repeated:
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_LENGTH_DELIMITED)
tag_bytes = encoder.TagBytes(
field_number, wire_format.WIRETYPE_LENGTH_DELIMITED
)
tag_len = len(tag_bytes)
def DecodeRepeatedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -651,7 +693,7 @@ def BytesDecoder(field_number, is_repeated, is_packed, key, new_default,
if value is None:
value = field_dict.setdefault(key, new_default(message))
while 1:
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated string.')
@ -667,7 +709,7 @@ def BytesDecoder(field_number, is_repeated, is_packed, key, new_default,
def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
del current_depth # unused
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated string.')
@ -683,15 +725,14 @@ def BytesDecoder(field_number, is_repeated, is_packed, key, new_default,
def GroupDecoder(field_number, is_repeated, is_packed, key, new_default):
"""Returns a decoder for a group field."""
end_tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_END_GROUP)
end_tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_END_GROUP)
end_tag_len = len(end_tag_bytes)
assert not is_packed
if is_repeated:
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_START_GROUP)
tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_START_GROUP)
tag_len = len(tag_bytes)
def DecodeRepeatedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -711,7 +752,7 @@ def GroupDecoder(field_number, is_repeated, is_packed, key, new_default):
pos = value.add()._InternalParse(buffer, pos, end, current_depth)
current_depth -= 1
# Read end tag.
new_pos = pos+end_tag_len
new_pos = pos + end_tag_len
if buffer[pos:new_pos] != end_tag_bytes or new_pos > end:
raise _DecodeError('Missing group end tag.')
# Predict that the next tag is another copy of the same repeated field.
@ -734,7 +775,7 @@ def GroupDecoder(field_number, is_repeated, is_packed, key, new_default):
pos = value._InternalParse(buffer, pos, end, current_depth)
current_depth -= 1
# Read end tag.
new_pos = pos+end_tag_len
new_pos = pos + end_tag_len
if buffer[pos:new_pos] != end_tag_bytes or new_pos > end:
raise _DecodeError('Missing group end tag.')
return new_pos
@ -749,9 +790,11 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
assert not is_packed
if is_repeated:
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_LENGTH_DELIMITED)
tag_bytes = encoder.TagBytes(
field_number, wire_format.WIRETYPE_LENGTH_DELIMITED
)
tag_len = len(tag_bytes)
def DecodeRepeatedField(
buffer, pos, end, message, field_dict, current_depth=0
):
@ -760,7 +803,7 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
value = field_dict.setdefault(key, new_default(message))
while 1:
# Read length.
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated message.')
@ -792,7 +835,7 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
if value is None:
value = field_dict.setdefault(key, new_default(message))
# Read length.
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated message.')
@ -814,6 +857,7 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
MESSAGE_SET_ITEM_TAG = encoder.TagBytes(1, wire_format.WIRETYPE_START_GROUP)
def MessageSetItemDecoder(descriptor):
"""Returns a decoder for a MessageSet item.
@ -856,11 +900,11 @@ def MessageSetItemDecoder(descriptor):
# Technically, type_id and message can appear in any order, so we need
# a little loop here.
while 1:
(tag_bytes, pos) = local_ReadTag(buffer, pos)
tag_bytes, pos = local_ReadTag(buffer, pos)
if tag_bytes == type_id_tag_bytes:
(type_id, pos) = local_DecodeVarint(buffer, pos)
type_id, pos = local_DecodeVarint(buffer, pos)
elif tag_bytes == message_tag_bytes:
(size, message_start) = local_DecodeVarint(buffer, pos)
size, message_start = local_DecodeVarint(buffer, pos)
pos = message_end = message_start + size
elif tag_bytes == item_end_tag_bytes:
break
@ -886,8 +930,7 @@ def MessageSetItemDecoder(descriptor):
message_type = extension.message_type
if not hasattr(message_type, '_concrete_class'):
message_factory.GetMessageClass(message_type)
value = field_dict.setdefault(
extension, message_type._concrete_class())
value = field_dict.setdefault(extension, message_type._concrete_class())
current_depth += 1
if current_depth > _recursion_limit:
raise _DecodeError('Error parsing message: too many levels of nesting.')
@ -905,7 +948,8 @@ def MessageSetItemDecoder(descriptor):
if not message._unknown_fields:
message._unknown_fields = []
message._unknown_fields.append(
(MESSAGE_SET_ITEM_TAG, buffer[message_set_item_start:pos].tobytes()))
(MESSAGE_SET_ITEM_TAG, buffer[message_set_item_start:pos].tobytes())
)
# pylint: enable=protected-access
return pos
@ -926,11 +970,11 @@ def UnknownMessageSetItemDecoder():
message_start = -1
message_end = -1
while 1:
(tag_bytes, pos) = ReadTag(buffer, pos)
tag_bytes, pos = ReadTag(buffer, pos)
if tag_bytes == type_id_tag_bytes:
(type_id, pos) = _DecodeVarint(buffer, pos)
type_id, pos = _DecodeVarint(buffer, pos)
elif tag_bytes == message_tag_bytes:
(size, message_start) = _DecodeVarint(buffer, pos)
size, message_start = _DecodeVarint(buffer, pos)
pos = message_end = message_start + size
elif tag_bytes == item_end_tag_bytes:
break
@ -952,14 +996,17 @@ def UnknownMessageSetItemDecoder():
return DecodeUnknownItem
# --------------------------------------------------------------------
def MapDecoder(field_descriptor, new_default, is_message_map):
"""Returns a decoder for a map field."""
key = field_descriptor
tag_bytes = encoder.TagBytes(field_descriptor.number,
wire_format.WIRETYPE_LENGTH_DELIMITED)
tag_bytes = encoder.TagBytes(
field_descriptor.number, wire_format.WIRETYPE_LENGTH_DELIMITED
)
tag_len = len(tag_bytes)
local_DecodeVarint = _DecodeVarint
# Can't read _concrete_class yet; might not be initialized.
@ -972,7 +1019,7 @@ def MapDecoder(field_descriptor, new_default, is_message_map):
value = field_dict.setdefault(key, new_default(message))
while 1:
# Read length.
(size, pos) = local_DecodeVarint(buffer, pos)
size, pos = local_DecodeVarint(buffer, pos)
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated message.')
@ -1012,6 +1059,8 @@ def _DecodeFixed32(buffer, pos):
new_pos = pos + 4
return (struct.unpack('<I', buffer[pos:new_pos])[0], new_pos)
DEFAULT_RECURSION_LIMIT = 100
_recursion_limit = DEFAULT_RECURSION_LIMIT
@ -1026,12 +1075,12 @@ def _DecodeUnknownFieldSet(buffer, pos, end_pos=None, current_depth=0):
unknown_field_set = containers.UnknownFieldSet()
while end_pos is None or pos < end_pos:
(tag_bytes, pos) = ReadTag(buffer, pos)
(tag, _) = _DecodeVarint(tag_bytes, 0)
tag_bytes, pos = ReadTag(buffer, pos)
tag, _ = _DecodeVarint(tag_bytes, 0)
field_number, wire_type = wire_format.UnpackTag(tag)
if wire_type == wire_format.WIRETYPE_END_GROUP:
break
(data, pos) = _DecodeUnknownField(
data, pos = _DecodeUnknownField(
buffer, pos, end_pos, field_number, wire_type, current_depth
)
# pylint: disable=protected-access
@ -1046,14 +1095,14 @@ def _DecodeUnknownField(
"""Decode a unknown field. Returns the UnknownField and new position."""
if wire_type == wire_format.WIRETYPE_VARINT:
(data, pos) = _DecodeVarint(buffer, pos)
data, pos = _DecodeVarint(buffer, pos)
elif wire_type == wire_format.WIRETYPE_FIXED64:
(data, pos) = _DecodeFixed64(buffer, pos)
data, pos = _DecodeFixed64(buffer, pos)
elif wire_type == wire_format.WIRETYPE_FIXED32:
(data, pos) = _DecodeFixed32(buffer, pos)
data, pos = _DecodeFixed32(buffer, pos)
elif wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED:
(size, pos) = _DecodeVarint(buffer, pos)
data = buffer[pos:pos+size].tobytes()
size, pos = _DecodeVarint(buffer, pos)
data = buffer[pos : pos + size].tobytes()
pos += size
elif wire_type == wire_format.WIRETYPE_START_GROUP:
end_tag_bytes = encoder.TagBytes(

5
python/google/protobuf/internal/decoder_test.py
View file

@ -20,7 +20,6 @@ from google.protobuf.internal import wire_format
from absl.testing import parameterized
_INPUT_BYTES = b'\x84r\x12'
_EXPECTED = (14596, 18)
@ -29,11 +28,11 @@ _EXPECTED = (14596, 18)
class DecoderTest(parameterized.TestCase):
def test_decode_varint_bytes(self):
(size, pos) = decoder._DecodeVarint(_INPUT_BYTES, 0)
size, pos = decoder._DecodeVarint(_INPUT_BYTES, 0)
self.assertEqual(size, _EXPECTED[0])
self.assertEqual(pos, 2)
(size, pos) = decoder._DecodeVarint(_INPUT_BYTES, 2)
size, pos = decoder._DecodeVarint(_INPUT_BYTES, 2)
self.assertEqual(size, _EXPECTED[1])
self.assertEqual(pos, 3)

129
python/google/protobuf/internal/descriptor_database_test.py
View file

@ -12,11 +12,12 @@ __author__ = 'matthewtoia@google.com (Matt Toia)'
import unittest
import warnings
from google.protobuf import descriptor_database
from google.protobuf import descriptor_pb2
from google.protobuf.internal import factory_test2_pb2
from google.protobuf.internal import no_package_pb2
from google.protobuf.internal import testing_refleaks
from google.protobuf import descriptor_database
from google.protobuf import unittest_pb2
@ -26,36 +27,74 @@ class DescriptorDatabaseTest(unittest.TestCase):
def testAdd(self):
db = descriptor_database.DescriptorDatabase()
file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString(
factory_test2_pb2.DESCRIPTOR.serialized_pb)
factory_test2_pb2.DESCRIPTOR.serialized_pb
)
file_desc_proto2 = descriptor_pb2.FileDescriptorProto.FromString(
no_package_pb2.DESCRIPTOR.serialized_pb)
no_package_pb2.DESCRIPTOR.serialized_pb
)
db.Add(file_desc_proto)
db.Add(file_desc_proto2)
self.assertEqual(file_desc_proto, db.FindFileByName(
'google/protobuf/internal/factory_test2.proto'))
self.assertEqual(
file_desc_proto,
db.FindFileByName(
'google/protobuf/internal/factory_test2.proto'
),
)
# Can find message type.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message'
),
)
# Can find nested message type.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.NestedFactory2Message'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.NestedFactory2Message'
),
)
# Can find enum type.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Enum'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Enum'
),
)
# Can find nested enum type.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum'))
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.MessageWithNestedEnumOnly.NestedEnum'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum'
),
)
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.MessageWithNestedEnumOnly.NestedEnum'
),
)
# Can find field.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.list_field'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.list_field'
),
)
# Can find enum value.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Enum.FACTORY_2_VALUE_0'))
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.FACTORY_2_VALUE_0'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Enum.FACTORY_2_VALUE_0'
),
)
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.FACTORY_2_VALUE_0'
),
)
self.assertEqual(
file_desc_proto2, db.FindFileContainingSymbol('NO_PACKAGE_VALUE_0')
)
@ -77,23 +116,36 @@ class DescriptorDatabaseTest(unittest.TestCase):
db.FindFileContainingSymbol('.NoPackageEnum'),
)
# Can find top level extension.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.another_field'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.another_field'
),
)
# Can find nested extension inside a message.
self.assertEqual(file_desc_proto, db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.one_more_field'))
self.assertEqual(
file_desc_proto,
db.FindFileContainingSymbol(
'google.protobuf.python.internal.Factory2Message.one_more_field'
),
)
# Can find service.
file_desc_proto2 = descriptor_pb2.FileDescriptorProto.FromString(
unittest_pb2.DESCRIPTOR.serialized_pb)
unittest_pb2.DESCRIPTOR.serialized_pb
)
db.Add(file_desc_proto2)
self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol(
'proto2_unittest.TestService'))
self.assertEqual(
file_desc_proto2,
db.FindFileContainingSymbol('proto2_unittest.TestService'),
)
# Non-existent field under a valid top level symbol can also be
# found. The behavior is the same with protobuf C++.
self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol(
'proto2_unittest.TestAllTypes.none_field'))
self.assertEqual(
file_desc_proto2,
db.FindFileContainingSymbol('proto2_unittest.TestAllTypes.none_field'),
)
with self.assertRaisesRegex(KeyError, r'\'proto2_unittest\.NoneMessage\''):
db.FindFileContainingSymbol('proto2_unittest.NoneMessage')
@ -106,10 +158,12 @@ class DescriptorDatabaseTest(unittest.TestCase):
def testConflictRegister(self):
db = descriptor_database.DescriptorDatabase()
unittest_fd = descriptor_pb2.FileDescriptorProto.FromString(
unittest_pb2.DESCRIPTOR.serialized_pb)
unittest_pb2.DESCRIPTOR.serialized_pb
)
db.Add(unittest_fd)
conflict_fd = descriptor_pb2.FileDescriptorProto.FromString(
unittest_pb2.DESCRIPTOR.serialized_pb)
unittest_pb2.DESCRIPTOR.serialized_pb
)
conflict_fd.name = 'other_file2'
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
@ -117,11 +171,14 @@ class DescriptorDatabaseTest(unittest.TestCase):
db.Add(conflict_fd)
self.assertTrue(len(w))
self.assertIs(w[0].category, RuntimeWarning)
self.assertIn('Conflict register for file "other_file2": ',
str(w[0].message))
self.assertIn(
'already defined in file '
'"google/protobuf/unittest.proto"', str(w[0].message))
'Conflict register for file "other_file2": ', str(w[0].message)
)
self.assertIn(
'already defined in file "google/protobuf/unittest.proto"',
str(w[0].message),
)
if __name__ == '__main__':
unittest.main()

791
python/google/protobuf/internal/descriptor_pool_test.py
View file

File diff suppressed because it is too large Load diff

798
python/google/protobuf/internal/descriptor_test.py
View file

File diff suppressed because it is too large Load diff

241
python/google/protobuf/internal/encoder.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Code for encoding protocol message primitives.
Contains the logic for encoding every logical protocol field type
@ -47,7 +46,6 @@ import struct
from google.protobuf.internal import wire_format
# This will overflow and thus become IEEE-754 "infinity". We would use
# "float('inf')" but it doesn't work on Windows pre-Python-2.6.
_POS_INF = 1e10000
@ -56,36 +54,57 @@ _NEG_INF = -_POS_INF
def _VarintSize(value):
"""Compute the size of a varint value."""
if value <= 0x7f: return 1
if value <= 0x3fff: return 2
if value <= 0x1fffff: return 3
if value <= 0xfffffff: return 4
if value <= 0x7ffffffff: return 5
if value <= 0x3ffffffffff: return 6
if value <= 0x1ffffffffffff: return 7
if value <= 0xffffffffffffff: return 8
if value <= 0x7fffffffffffffff: return 9
if value <= 0x7F:
return 1
if value <= 0x3FFF:
return 2
if value <= 0x1FFFFF:
return 3
if value <= 0xFFFFFFF:
return 4
if value <= 0x7FFFFFFFF:
return 5
if value <= 0x3FFFFFFFFFF:
return 6
if value <= 0x1FFFFFFFFFFFF:
return 7
if value <= 0xFFFFFFFFFFFFFF:
return 8
if value <= 0x7FFFFFFFFFFFFFFF:
return 9
return 10
def _SignedVarintSize(value):
"""Compute the size of a signed varint value."""
if value < 0: return 10
if value <= 0x7f: return 1
if value <= 0x3fff: return 2
if value <= 0x1fffff: return 3
if value <= 0xfffffff: return 4
if value <= 0x7ffffffff: return 5
if value <= 0x3ffffffffff: return 6
if value <= 0x1ffffffffffff: return 7
if value <= 0xffffffffffffff: return 8
if value <= 0x7fffffffffffffff: return 9
if value < 0:
return 10
if value <= 0x7F:
return 1
if value <= 0x3FFF:
return 2
if value <= 0x1FFFFF:
return 3
if value <= 0xFFFFFFF:
return 4
if value <= 0x7FFFFFFFF:
return 5
if value <= 0x3FFFFFFFFFF:
return 6
if value <= 0x1FFFFFFFFFFFF:
return 7
if value <= 0xFFFFFFFFFFFFFF:
return 8
if value <= 0x7FFFFFFFFFFFFFFF:
return 9
return 10
def _TagSize(field_number):
"""Returns the number of bytes required to serialize a tag with this field
number."""
number.
"""
# Just pass in type 0, since the type won't affect the tag+type size.
return _VarintSize(wire_format.PackTag(field_number, 0))
@ -100,28 +119,36 @@ def _TagSize(field_number):
def _SimpleSizer(compute_value_size):
"""A sizer which uses the function compute_value_size to compute the size of
each value. Typically compute_value_size is _VarintSize."""
each value. Typically compute_value_size is _VarintSize.
"""
def SpecificSizer(field_number, is_repeated, is_packed):
tag_size = _TagSize(field_number)
if is_packed:
local_VarintSize = _VarintSize
def PackedFieldSize(value):
result = 0
for element in value:
result += compute_value_size(element)
return result + local_VarintSize(result) + tag_size
return PackedFieldSize
elif is_repeated:
def RepeatedFieldSize(value):
result = tag_size * len(value)
for element in value:
result += compute_value_size(element)
return result
return RepeatedFieldSize
else:
def FieldSize(value):
return tag_size + compute_value_size(value)
return FieldSize
return SpecificSizer
@ -129,54 +156,70 @@ def _SimpleSizer(compute_value_size):
def _ModifiedSizer(compute_value_size, modify_value):
"""Like SimpleSizer, but modify_value is invoked on each value before it is
passed to compute_value_size. modify_value is typically ZigZagEncode."""
passed to compute_value_size. modify_value is typically ZigZagEncode.
"""
def SpecificSizer(field_number, is_repeated, is_packed):
tag_size = _TagSize(field_number)
if is_packed:
local_VarintSize = _VarintSize
def PackedFieldSize(value):
result = 0
for element in value:
result += compute_value_size(modify_value(element))
return result + local_VarintSize(result) + tag_size
return PackedFieldSize
elif is_repeated:
def RepeatedFieldSize(value):
result = tag_size * len(value)
for element in value:
result += compute_value_size(modify_value(element))
return result
return RepeatedFieldSize
else:
def FieldSize(value):
return tag_size + compute_value_size(modify_value(value))
return FieldSize
return SpecificSizer
def _FixedSizer(value_size):
"""Like _SimpleSizer except for a fixed-size field. The input is the size
of one value."""
"""Like _SimpleSizer except for a fixed-size field.
The input is the size of one value.
"""
def SpecificSizer(field_number, is_repeated, is_packed):
tag_size = _TagSize(field_number)
if is_packed:
local_VarintSize = _VarintSize
def PackedFieldSize(value):
result = len(value) * value_size
return result + local_VarintSize(result) + tag_size
return PackedFieldSize
elif is_repeated:
element_size = value_size + tag_size
def RepeatedFieldSize(value):
return len(value) * element_size
return RepeatedFieldSize
else:
field_size = value_size + tag_size
def FieldSize(value):
return field_size
return FieldSize
return SpecificSizer
@ -188,15 +231,15 @@ def _FixedSizer(value_size):
# as parameters and returns a sizer, which in turn takes a field value as
# a parameter and returns its encoded size.
Int32Sizer = Int64Sizer = EnumSizer = _SimpleSizer(_SignedVarintSize)
UInt32Sizer = UInt64Sizer = _SimpleSizer(_VarintSize)
SInt32Sizer = SInt64Sizer = _ModifiedSizer(
_SignedVarintSize, wire_format.ZigZagEncode)
_SignedVarintSize, wire_format.ZigZagEncode
)
Fixed32Sizer = SFixed32Sizer = FloatSizer = _FixedSizer(4)
Fixed32Sizer = SFixed32Sizer = FloatSizer = _FixedSizer(4)
Fixed64Sizer = SFixed64Sizer = DoubleSizer = _FixedSizer(8)
BoolSizer = _FixedSizer(1)
@ -210,17 +253,21 @@ def StringSizer(field_number, is_repeated, is_packed):
local_len = len
assert not is_packed
if is_repeated:
def RepeatedFieldSize(value):
result = tag_size * len(value)
for element in value:
l = local_len(element.encode('utf-8'))
result += local_VarintSize(l) + l
return result
return RepeatedFieldSize
else:
def FieldSize(value):
l = local_len(value.encode('utf-8'))
return tag_size + local_VarintSize(l) + l
return FieldSize
@ -232,17 +279,21 @@ def BytesSizer(field_number, is_repeated, is_packed):
local_len = len
assert not is_packed
if is_repeated:
def RepeatedFieldSize(value):
result = tag_size * len(value)
for element in value:
l = local_len(element)
result += local_VarintSize(l) + l
return result
return RepeatedFieldSize
else:
def FieldSize(value):
l = local_len(value)
return tag_size + local_VarintSize(l) + l
return FieldSize
@ -252,15 +303,19 @@ def GroupSizer(field_number, is_repeated, is_packed):
tag_size = _TagSize(field_number) * 2
assert not is_packed
if is_repeated:
def RepeatedFieldSize(value):
result = tag_size * len(value)
for element in value:
result += element.ByteSize()
return result
return RepeatedFieldSize
else:
def FieldSize(value):
return tag_size + value.ByteSize()
return FieldSize
@ -271,17 +326,21 @@ def MessageSizer(field_number, is_repeated, is_packed):
local_VarintSize = _VarintSize
assert not is_packed
if is_repeated:
def RepeatedFieldSize(value):
result = tag_size * len(value)
for element in value:
l = element.ByteSize()
result += local_VarintSize(l) + l
return result
return RepeatedFieldSize
else:
def FieldSize(value):
l = value.ByteSize()
return tag_size + local_VarintSize(l) + l
return FieldSize
@ -300,8 +359,9 @@ def MessageSetItemSizer(field_number):
}
}
"""
static_size = (_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) +
_TagSize(3))
static_size = (
_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) + _TagSize(3)
)
local_VarintSize = _VarintSize
def FieldSize(value):
@ -340,6 +400,7 @@ def MapSizer(field_descriptor, is_message_map):
return FieldSize
# ====================================================================
# Encoders!
@ -350,11 +411,11 @@ def _VarintEncoder():
local_int2byte = struct.Struct('>B').pack
def EncodeVarint(write, value, unused_deterministic=None):
bits = value & 0x7f
bits = value & 0x7F
value >>= 7
while value:
write(local_int2byte(0x80|bits))
bits = value & 0x7f
write(local_int2byte(0x80 | bits))
bits = value & 0x7F
value >>= 7
return write(local_int2byte(bits))
@ -363,18 +424,20 @@ def _VarintEncoder():
def _SignedVarintEncoder():
"""Return an encoder for a basic signed varint value (does not include
tag)."""
tag).
"""
local_int2byte = struct.Struct('>B').pack
def EncodeSignedVarint(write, value, unused_deterministic=None):
if value < 0:
value += (1 << 64)
bits = value & 0x7f
value += 1 << 64
bits = value & 0x7F
value >>= 7
while value:
write(local_int2byte(0x80|bits))
bits = value & 0x7f
write(local_int2byte(0x80 | bits))
bits = value & 0x7F
value >>= 7
return write(local_int2byte(bits))
@ -386,12 +449,14 @@ _EncodeSignedVarint = _SignedVarintEncoder()
def _VarintBytes(value):
"""Encode the given integer as a varint and return the bytes. This is only
called at startup time so it doesn't need to be fast."""
"""Encode the given integer as a varint and return the bytes.
This is only called at startup time so it doesn't need to be fast.
"""
pieces = []
_EncodeVarint(pieces.append, value, True)
return b"".join(pieces)
return b''.join(pieces)
def TagBytes(field_number, wire_type):
@ -399,6 +464,7 @@ def TagBytes(field_number, wire_type):
return bytes(_VarintBytes(wire_format.PackTag(field_number, wire_type)))
# --------------------------------------------------------------------
# As with sizers (see above), we have a number of common encoder
# implementations.
@ -419,6 +485,7 @@ def _SimpleEncoder(wire_type, encode_value, compute_value_size):
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value, deterministic):
write(tag_bytes)
size = 0
@ -427,19 +494,24 @@ def _SimpleEncoder(wire_type, encode_value, compute_value_size):
local_EncodeVarint(write, size, deterministic)
for element in value:
encode_value(write, element, deterministic)
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeRepeatedField(write, value, deterministic):
for element in value:
write(tag_bytes)
encode_value(write, element, deterministic)
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeField(write, value, deterministic):
write(tag_bytes)
return encode_value(write, value, deterministic)
return EncodeField
return SpecificEncoder
@ -447,12 +519,15 @@ def _SimpleEncoder(wire_type, encode_value, compute_value_size):
def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value):
"""Like SimpleEncoder but additionally invokes modify_value on every value
before passing it to encode_value. Usually modify_value is ZigZagEncode."""
before passing it to encode_value. Usually modify_value is ZigZagEncode.
"""
def SpecificEncoder(field_number, is_repeated, is_packed):
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value, deterministic):
write(tag_bytes)
size = 0
@ -461,19 +536,24 @@ def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value):
local_EncodeVarint(write, size, deterministic)
for element in value:
encode_value(write, modify_value(element), deterministic)
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeRepeatedField(write, value, deterministic):
for element in value:
write(tag_bytes)
encode_value(write, modify_value(element), deterministic)
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeField(write, value, deterministic):
write(tag_bytes)
return encode_value(write, modify_value(value), deterministic)
return EncodeField
return SpecificEncoder
@ -494,24 +574,30 @@ def _StructPackEncoder(wire_type, format):
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value, deterministic):
write(tag_bytes)
local_EncodeVarint(write, len(value) * value_size, deterministic)
for element in value:
write(local_struct_pack(format, element))
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeRepeatedField(write, value, unused_deterministic=None):
for element in value:
write(tag_bytes)
write(local_struct_pack(format, element))
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeField(write, value, unused_deterministic=None):
write(tag_bytes)
return write(local_struct_pack(format, value))
return EncodeField
return SpecificEncoder
@ -531,35 +617,42 @@ def _FloatingPointEncoder(wire_type, format):
value_size = struct.calcsize(format)
if value_size == 4:
def EncodeNonFiniteOrRaise(write, value):
# Remember that the serialized form uses little-endian byte order.
if value == _POS_INF:
write(b'\x00\x00\x80\x7F')
elif value == _NEG_INF:
write(b'\x00\x00\x80\xFF')
elif value != value: # NaN
elif value != value: # NaN
write(b'\x00\x00\xC0\x7F')
else:
raise
elif value_size == 8:
def EncodeNonFiniteOrRaise(write, value):
if value == _POS_INF:
write(b'\x00\x00\x00\x00\x00\x00\xF0\x7F')
elif value == _NEG_INF:
write(b'\x00\x00\x00\x00\x00\x00\xF0\xFF')
elif value != value: # NaN
elif value != value: # NaN
write(b'\x00\x00\x00\x00\x00\x00\xF8\x7F')
else:
raise
else:
raise ValueError('Can\'t encode floating-point values that are '
'%d bytes long (only 4 or 8)' % value_size)
raise ValueError(
"Can't encode floating-point values that are "
'%d bytes long (only 4 or 8)' % value_size
)
def SpecificEncoder(field_number, is_repeated, is_packed):
local_struct_pack = struct.pack
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value, deterministic):
write(tag_bytes)
local_EncodeVarint(write, len(value) * value_size, deterministic)
@ -570,9 +663,11 @@ def _FloatingPointEncoder(wire_type, format):
write(local_struct_pack(format, element))
except SystemError:
EncodeNonFiniteOrRaise(write, element)
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeRepeatedField(write, value, unused_deterministic=None):
for element in value:
write(tag_bytes)
@ -580,15 +675,18 @@ def _FloatingPointEncoder(wire_type, format):
write(local_struct_pack(format, element))
except SystemError:
EncodeNonFiniteOrRaise(write, element)
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_type)
def EncodeField(write, value, unused_deterministic=None):
write(tag_bytes)
try:
write(local_struct_pack(format, value))
except SystemError:
EncodeNonFiniteOrRaise(write, value)
return EncodeField
return SpecificEncoder
@ -598,27 +696,31 @@ def _FloatingPointEncoder(wire_type, format):
# Here we declare an encoder constructor for each field type. These work
# very similarly to sizer constructors, described earlier.
Int32Encoder = Int64Encoder = EnumEncoder = _SimpleEncoder(
wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize)
wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize
)
UInt32Encoder = UInt64Encoder = _SimpleEncoder(
wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize)
wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize
)
SInt32Encoder = SInt64Encoder = _ModifiedEncoder(
wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize,
wire_format.ZigZagEncode)
wire_format.WIRETYPE_VARINT,
_EncodeVarint,
_VarintSize,
wire_format.ZigZagEncode,
)
# Note that Python conveniently guarantees that when using the '<' prefix on
# formats, they will also have the same size across all platforms (as opposed
# to without the prefix, where their sizes depend on the C compiler's basic
# type sizes).
Fixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I')
Fixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q')
Fixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I')
Fixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q')
SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<i')
SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<q')
FloatEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f')
DoubleEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d')
FloatEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f')
DoubleEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d')
def BoolEncoder(field_number, is_repeated, is_packed):
@ -629,6 +731,7 @@ def BoolEncoder(field_number, is_repeated, is_packed):
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
def EncodePackedField(write, value, deterministic):
write(tag_bytes)
local_EncodeVarint(write, len(value), deterministic)
@ -637,9 +740,11 @@ def BoolEncoder(field_number, is_repeated, is_packed):
write(true_byte)
else:
write(false_byte)
return EncodePackedField
elif is_repeated:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
def EncodeRepeatedField(write, value, unused_deterministic=None):
for element in value:
write(tag_bytes)
@ -647,14 +752,17 @@ def BoolEncoder(field_number, is_repeated, is_packed):
write(true_byte)
else:
write(false_byte)
return EncodeRepeatedField
else:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
def EncodeField(write, value, unused_deterministic=None):
write(tag_bytes)
if value:
return write(true_byte)
return write(false_byte)
return EncodeField
@ -666,19 +774,23 @@ def StringEncoder(field_number, is_repeated, is_packed):
local_len = len
assert not is_packed
if is_repeated:
def EncodeRepeatedField(write, value, deterministic):
for element in value:
encoded = element.encode('utf-8')
write(tag)
local_EncodeVarint(write, local_len(encoded), deterministic)
write(encoded)
return EncodeRepeatedField
else:
def EncodeField(write, value, deterministic):
encoded = value.encode('utf-8')
write(tag)
local_EncodeVarint(write, local_len(encoded), deterministic)
return write(encoded)
return EncodeField
@ -690,17 +802,21 @@ def BytesEncoder(field_number, is_repeated, is_packed):
local_len = len
assert not is_packed
if is_repeated:
def EncodeRepeatedField(write, value, deterministic):
for element in value:
write(tag)
local_EncodeVarint(write, local_len(element), deterministic)
write(element)
return EncodeRepeatedField
else:
def EncodeField(write, value, deterministic):
write(tag)
local_EncodeVarint(write, local_len(value), deterministic)
return write(value)
return EncodeField
@ -711,17 +827,21 @@ def GroupEncoder(field_number, is_repeated, is_packed):
end_tag = TagBytes(field_number, wire_format.WIRETYPE_END_GROUP)
assert not is_packed
if is_repeated:
def EncodeRepeatedField(write, value, deterministic):
for element in value:
write(start_tag)
element._InternalSerialize(write, deterministic)
write(end_tag)
return EncodeRepeatedField
else:
def EncodeField(write, value, deterministic):
write(start_tag)
value._InternalSerialize(write, deterministic)
return write(end_tag)
return EncodeField
@ -732,17 +852,21 @@ def MessageEncoder(field_number, is_repeated, is_packed):
local_EncodeVarint = _EncodeVarint
assert not is_packed
if is_repeated:
def EncodeRepeatedField(write, value, deterministic):
for element in value:
write(tag)
local_EncodeVarint(write, element.ByteSize(), deterministic)
element._InternalSerialize(write, deterministic)
return EncodeRepeatedField
else:
def EncodeField(write, value, deterministic):
write(tag)
local_EncodeVarint(write, value.ByteSize(), deterministic)
return value._InternalSerialize(write, deterministic)
return EncodeField
@ -761,11 +885,12 @@ def MessageSetItemEncoder(field_number):
}
}
"""
start_bytes = b"".join([
start_bytes = b''.join([
TagBytes(1, wire_format.WIRETYPE_START_GROUP),
TagBytes(2, wire_format.WIRETYPE_VARINT),
_VarintBytes(field_number),
TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)])
TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED),
])
end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP)
local_EncodeVarint = _EncodeVarint

52
python/google/protobuf/internal/enum_type_wrapper.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""A simple wrapper around enum types to expose utility functions.
Instances are created as properties with the same name as the enum they wrap
@ -17,7 +16,6 @@ from typing import TYPE_CHECKING
__author__ = 'rabsatt@google.com (Kevin Rabsatt)'
if TYPE_CHECKING:
# EnumTypeWrapper is used as a metaclass during type checking, specifically
# in the generated stub files.
@ -55,11 +53,16 @@ class EnumTypeWrapper(base):
if not isinstance(number, int):
raise TypeError(
'Enum value for {} must be an int, but got {} {!r}.'.format(
self._enum_type.name, type(number), number))
self._enum_type.name, type(number), number
)
)
else:
# repr here to handle the odd case when you pass in a boolean.
raise ValueError('Enum {} has no name defined for value {!r}'.format(
self._enum_type.name, number))
raise ValueError(
'Enum {} has no name defined for value {!r}'.format(
self._enum_type.name, number
)
)
def Value(self, name): # pylint: disable=invalid-name
"""Returns the value corresponding to the given enum name."""
@ -67,8 +70,11 @@ class EnumTypeWrapper(base):
return self._enum_type.values_by_name[name].number
except KeyError:
pass # fall out to break exception chaining
raise ValueError('Enum {} has no value defined for name {!r}'.format(
self._enum_type.name, name))
raise ValueError(
'Enum {} has no value defined for name {!r}'.format(
self._enum_type.name, name
)
)
def keys(self):
"""Return a list of the string names in the enum.
@ -77,8 +83,9 @@ class EnumTypeWrapper(base):
A list of strs, in the order they were defined in the .proto file.
"""
return [value_descriptor.name
for value_descriptor in self._enum_type.values]
return [
value_descriptor.name for value_descriptor in self._enum_type.values
]
def values(self):
"""Return a list of the integer values in the enum.
@ -87,8 +94,9 @@ class EnumTypeWrapper(base):
A list of ints, in the order they were defined in the .proto file.
"""
return [value_descriptor.number
for value_descriptor in self._enum_type.values]
return [
value_descriptor.number for value_descriptor in self._enum_type.values
]
def items(self):
"""Return a list of the (name, value) pairs of the enum.
@ -97,19 +105,27 @@ class EnumTypeWrapper(base):
A list of (str, int) pairs, in the order they were defined
in the .proto file.
"""
return [(value_descriptor.name, value_descriptor.number)
for value_descriptor in self._enum_type.values]
return [
(value_descriptor.name, value_descriptor.number)
for value_descriptor in self._enum_type.values
]
def __getattr__(self, name):
"""Returns the value corresponding to the given enum name."""
try:
return super(
EnumTypeWrapper,
self).__getattribute__('_enum_type').values_by_name[name].number
return (
super(EnumTypeWrapper, self)
.__getattribute__('_enum_type')
.values_by_name[name]
.number
)
except KeyError:
pass # fall out to break exception chaining
raise AttributeError('Enum {} has no value defined for name {!r}'.format(
self._enum_type.name, name))
raise AttributeError(
'Enum {} has no value defined for name {!r}'.format(
self._enum_type.name, name
)
)
def __or__(self, other):
"""Returns the union type of self and other."""

63
python/google/protobuf/internal/extension_dict.py
View file

@ -4,41 +4,44 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains _ExtensionDict class to represent extensions."""
"""Contains _ExtensionDict class to represent extensions.
"""
from google.protobuf.internal import type_checkers
from google.protobuf.descriptor import FieldDescriptor
from google.protobuf.internal import type_checkers
def _VerifyExtensionHandle(message, extension_handle):
"""Verify that the given extension handle is valid."""
if not isinstance(extension_handle, FieldDescriptor):
raise KeyError('HasExtension() expects an extension handle, got: %s' %
extension_handle)
raise KeyError(
'HasExtension() expects an extension handle, got: %s' % extension_handle
)
if not extension_handle.is_extension:
raise KeyError('"%s" is not an extension.' % extension_handle.full_name)
if not extension_handle.containing_type:
raise KeyError('"%s" is missing a containing_type.'
% extension_handle.full_name)
raise KeyError(
'"%s" is missing a containing_type.' % extension_handle.full_name
)
if extension_handle.containing_type is not message.DESCRIPTOR:
raise KeyError('Extension "%s" extends message type "%s", but this '
'message is of type "%s".' %
(extension_handle.full_name,
extension_handle.containing_type.full_name,
message.DESCRIPTOR.full_name))
raise KeyError(
'Extension "%s" extends message type "%s", but this '
'message is of type "%s".'
% (
extension_handle.full_name,
extension_handle.containing_type.full_name,
message.DESCRIPTOR.full_name,
)
)
# TODO: Unify error handling of "unknown extension" crap.
# TODO: Support iteritems()-style iteration over all
# extensions with the "has" bits turned on?
class _ExtensionDict(object):
"""Dict-like container for Extension fields on proto instances.
Note that in all cases we expect extension handles to be
@ -46,9 +49,9 @@ class _ExtensionDict(object):
"""
def __init__(self, extended_message):
"""
Args:
extended_message: Message instance for which we are the Extensions dict.
"""Args:
extended_message: Message instance for which we are the Extensions dict.
"""
self._extended_message = extended_message
@ -68,9 +71,11 @@ class _ExtensionDict(object):
if not hasattr(message_type, '_concrete_class'):
# pylint: disable=g-import-not-at-top
from google.protobuf import message_factory
message_factory.GetMessageClass(message_type)
if not hasattr(extension_handle.message_type, '_concrete_class'):
from google.protobuf import message_factory
message_factory.GetMessageClass(extension_handle.message_type)
result = extension_handle.message_type._concrete_class()
try:
@ -88,8 +93,7 @@ class _ExtensionDict(object):
# WARNING: We are relying on setdefault() being atomic. This is true
# in CPython but we haven't investigated others. This warning appears
# in several other locations in this file.
result = self._extended_message._fields.setdefault(
extension_handle, result)
result = self._extended_message._fields.setdefault(extension_handle, result)
return result
@ -124,23 +128,29 @@ class _ExtensionDict(object):
# ancestors of the extended message.
def __setitem__(self, extension_handle, value):
"""If extension_handle specifies a non-repeated, scalar extension
field, sets the value of that field.
"""
_VerifyExtensionHandle(self._extended_message, extension_handle)
if (extension_handle.is_repeated or
extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE):
if (
extension_handle.is_repeated
or extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE
):
raise TypeError(
'Cannot assign to extension "%s" because it is a repeated or '
'composite type.' % extension_handle.full_name)
'composite type.'
% extension_handle.full_name
)
# It's slightly wasteful to lookup the type checker each time,
# but we expect this to be a vanishingly uncommon case anyway.
type_checker = type_checkers.GetTypeChecker(extension_handle)
# pylint: disable=protected-access
self._extended_message._fields[extension_handle] = (
type_checker.CheckValue(value))
self._extended_message._fields[extension_handle] = type_checker.CheckValue(
value
)
self._extended_message._Modified()
def __delitem__(self, extension_handle):
@ -174,8 +184,9 @@ class _ExtensionDict(object):
def __iter__(self):
# Return a generator over the populated extension fields
return (f[0] for f in self._extended_message.ListFields()
if f[0].is_extension)
return (
f[0] for f in self._extended_message.ListFields() if f[0].is_extension
)
def __contains__(self, extension_handle):
_VerifyExtensionHandle(self._extended_message, extension_handle)

100
python/google/protobuf/internal/field_mask.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains FieldMask class."""
from google.protobuf.descriptor import FieldDescriptor
@ -76,21 +75,26 @@ class FieldMask(object):
intersection.ToFieldMask(self)
def MergeMessage(
self, source, destination,
replace_message_field=False, replace_repeated_field=False):
self,
source,
destination,
replace_message_field=False,
replace_repeated_field=False,
):
"""Merges fields specified in FieldMask from source to destination.
Args:
source: Source message.
destination: The destination message to be merged into.
replace_message_field: Replace message field if True. Merge message
field if False.
replace_repeated_field: Replace repeated field if True. Append
elements of repeated field if False.
replace_message_field: Replace message field if True. Merge message field
if False.
replace_repeated_field: Replace repeated field if True. Append elements of
repeated field if False.
"""
tree = _FieldMaskTree(self)
tree.MergeMessage(
source, destination, replace_message_field, replace_repeated_field)
source, destination, replace_message_field, replace_repeated_field
)
def _IsValidPath(message_descriptor, path):
@ -99,9 +103,11 @@ def _IsValidPath(message_descriptor, path):
last = parts.pop()
for name in parts:
field = message_descriptor.fields_by_name.get(name)
if (field is None or
field.is_repeated or
field.type != FieldDescriptor.TYPE_MESSAGE):
if (
field is None
or field.is_repeated
or field.type != FieldDescriptor.TYPE_MESSAGE
):
return False
message_descriptor = field.message_type
return last in message_descriptor.fields_by_name
@ -110,10 +116,13 @@ def _IsValidPath(message_descriptor, path):
def _CheckFieldMaskMessage(message):
"""Raises ValueError if message is not a FieldMask."""
message_descriptor = message.DESCRIPTOR
if (message_descriptor.name != 'FieldMask' or
message_descriptor.file.name != 'google/protobuf/field_mask.proto'):
raise ValueError('Message {0} is not a FieldMask.'.format(
message_descriptor.full_name))
if (
message_descriptor.name != 'FieldMask'
or message_descriptor.file.name != 'google/protobuf/field_mask.proto'
):
raise ValueError(
'Message {0} is not a FieldMask.'.format(message_descriptor.full_name)
)
def _SnakeCaseToCamelCase(path_name):
@ -124,7 +133,8 @@ def _SnakeCaseToCamelCase(path_name):
if c.isupper():
raise ValueError(
'Fail to print FieldMask to Json string: Path name '
'{0} must not contain uppercase letters.'.format(path_name))
'{0} must not contain uppercase letters.'.format(path_name)
)
if after_underscore:
if c.islower():
result.append(c.upper())
@ -133,15 +143,18 @@ def _SnakeCaseToCamelCase(path_name):
raise ValueError(
'Fail to print FieldMask to Json string: The '
'character after a "_" must be a lowercase letter '
'in path name {0}.'.format(path_name))
'in path name {0}.'.format(path_name)
)
elif c == '_':
after_underscore = True
else:
result += c
if after_underscore:
raise ValueError('Fail to print FieldMask to Json string: Trailing "_" '
'in path name {0}.'.format(path_name))
raise ValueError(
'Fail to print FieldMask to Json string: Trailing "_" '
'in path name {0}.'.format(path_name)
)
return ''.join(result)
@ -150,8 +163,10 @@ def _CamelCaseToSnakeCase(path_name):
result = []
for c in path_name:
if c == '_':
raise ValueError('Fail to parse FieldMask: Path name '
'{0} must not contain "_"s.'.format(path_name))
raise ValueError(
'Fail to parse FieldMask: Path name '
'{0} must not contain "_"s.'.format(path_name)
)
if c.isupper():
result += '_'
result += c.lower()
@ -249,11 +264,12 @@ class _FieldMaskTree(object):
stack.append((child_path, current_node[name]))
def MergeMessage(
self, source, destination,
replace_message, replace_repeated):
self, source, destination, replace_message, replace_repeated
):
"""Merge all fields specified by this tree from source to destination."""
_MergeMessage(
self._root, source, destination, replace_message, replace_repeated)
self._root, source, destination, replace_message, replace_repeated
)
def _StrConvert(value):
@ -266,8 +282,7 @@ def _StrConvert(value):
return value
def _MergeMessage(
node, source, destination, replace_message, replace_repeated):
def _MergeMessage(node, source, destination, replace_message, replace_repeated):
"""Merge all fields specified by a sub-tree from source to destination."""
stack = [(node, source, destination)]
while stack:
@ -277,19 +292,29 @@ def _MergeMessage(
child = current_node[name]
field = source_descriptor.fields_by_name[name]
if field is None:
raise ValueError('Error: Can\'t find field {0} in message {1}.'.format(
name, source_descriptor.full_name))
raise ValueError(
"Error: Can't find field {0} in message {1}.".format(
name, source_descriptor.full_name
)
)
if child:
# Sub-paths are only allowed for singular message fields.
if (field.is_repeated or
field.cpp_type != FieldDescriptor.CPPTYPE_MESSAGE):
raise ValueError('Error: Field {0} in message {1} is not a singular '
'message field and cannot have sub-fields.'.format(
name, source_descriptor.full_name))
if (
field.is_repeated
or field.cpp_type != FieldDescriptor.CPPTYPE_MESSAGE
):
raise ValueError(
'Error: Field {0} in message {1} is not a singular '
'message field and cannot have sub-fields.'.format(
name, source_descriptor.full_name
)
)
if current_source.HasField(name):
stack.append(
(child, getattr(current_source, name),
getattr(current_destination, name)))
stack.append((
child,
getattr(current_source, name),
getattr(current_destination, name),
))
continue
if field.is_repeated:
if replace_repeated:
@ -303,7 +328,8 @@ def _MergeMessage(
current_destination.ClearField(_StrConvert(name))
if current_source.HasField(name):
getattr(current_destination, name).MergeFrom(
getattr(current_source, name))
getattr(current_source, name)
)
elif not field.has_presence or current_source.HasField(name):
setattr(current_destination, name, getattr(current_source, name))
else:

235
python/google/protobuf/internal/generator_test.py
View file

@ -9,25 +9,27 @@
# first, since it's testing the subtler code, and since it provides decent
# indirect testing of the protocol compiler output.
"""Unittest that directly tests the output of the pure-Python protocol
compiler. See //google/protobuf/internal/reflection_test.py for a test which
further ensures that we can use Python protocol message objects as we expect.
"""Unittest that directly tests the output of the pure-Python protocol compiler.
See //google/protobuf/internal/reflection_test.py for a test which further
ensures that we can use Python protocol message objects as we expect.
"""
__author__ = 'robinson@google.com (Will Robinson)'
import unittest
from google.protobuf.internal import test_bad_identifiers_pb2
from google.protobuf import symbol_database
from google.protobuf.internal import test_bad_identifiers_pb2
from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_import_public_pb2
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_mset_wire_format_pb2
from google.protobuf import unittest_no_generic_services_pb2
from google.protobuf import unittest_pb2
from google.protobuf import unittest_retention_pb2
from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_no_generic_services_pb2
MAX_EXTENSION = 536870912
@ -39,7 +41,8 @@ class GeneratorTest(unittest.TestCase):
proto_type = unittest_pb2.TestAllTypes
self.assertEqual(
proto_type.NestedMessage.DESCRIPTOR,
proto_type.DESCRIPTOR.fields_by_name[field_name].message_type)
proto_type.DESCRIPTOR.fields_by_name[field_name].message_type,
)
def testEnums(self):
# We test only module-level enums here.
@ -65,6 +68,7 @@ class GeneratorTest(unittest.TestCase):
def isnan(val):
# NaN is never equal to itself.
return val != val
def isinf(val):
# Infinity times zero equals NaN.
return not isnan(val) and isnan(val * 0)
@ -80,7 +84,7 @@ class GeneratorTest(unittest.TestCase):
self.assertTrue(isinf(message.neg_inf_float))
self.assertTrue(message.neg_inf_float < 0)
self.assertTrue(isnan(message.nan_float))
self.assertEqual("? ? ?? ?? ??? ??/ ??-", message.cpp_trigraph)
self.assertEqual('? ? ?? ?? ??? ??/ ??-', message.cpp_trigraph)
def testHasDefaultValues(self):
desc = unittest_pb2.TestAllTypes.DESCRIPTOR
@ -92,23 +96,31 @@ class GeneratorTest(unittest.TestCase):
'default_int32': True,
}
has_default_by_name = dict(
[(f.name, f.has_default_value)
for f in desc.fields
if f.name in expected_has_default_by_name])
has_default_by_name = dict([
(f.name, f.has_default_value)
for f in desc.fields
if f.name in expected_has_default_by_name
])
self.assertEqual(expected_has_default_by_name, has_default_by_name)
def testContainingTypeBehaviorForExtensions(self):
self.assertEqual(unittest_pb2.optional_int32_extension.containing_type,
unittest_pb2.TestAllExtensions.DESCRIPTOR)
self.assertEqual(unittest_pb2.TestRequired.single.containing_type,
unittest_pb2.TestAllExtensions.DESCRIPTOR)
self.assertEqual(
unittest_pb2.optional_int32_extension.containing_type,
unittest_pb2.TestAllExtensions.DESCRIPTOR,
)
self.assertEqual(
unittest_pb2.TestRequired.single.containing_type,
unittest_pb2.TestAllExtensions.DESCRIPTOR,
)
def testExtensionScope(self):
self.assertEqual(unittest_pb2.optional_int32_extension.extension_scope,
None)
self.assertEqual(unittest_pb2.TestRequired.single.extension_scope,
unittest_pb2.TestRequired.DESCRIPTOR)
self.assertEqual(
unittest_pb2.optional_int32_extension.extension_scope, None
)
self.assertEqual(
unittest_pb2.TestRequired.single.extension_scope,
unittest_pb2.TestRequired.DESCRIPTOR,
)
def testIsExtension(self):
self.assertTrue(unittest_pb2.optional_int32_extension.is_extension)
@ -212,99 +224,120 @@ class GeneratorTest(unittest.TestCase):
unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR,
unittest_pb2.TestAllTypes.OptionalGroup.DESCRIPTOR,
unittest_pb2.TestAllTypes.RepeatedGroup.DESCRIPTOR,
]))
]),
)
self.assertEqual(unittest_pb2.TestEmptyMessage.DESCRIPTOR.nested_types, [])
self.assertEqual(
unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.nested_types, [])
unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.nested_types, []
)
def testContainingType(self):
self.assertTrue(
unittest_pb2.TestEmptyMessage.DESCRIPTOR.containing_type is None)
unittest_pb2.TestEmptyMessage.DESCRIPTOR.containing_type is None
)
self.assertTrue(
unittest_pb2.TestAllTypes.DESCRIPTOR.containing_type is None)
unittest_pb2.TestAllTypes.DESCRIPTOR.containing_type is None
)
self.assertEqual(
unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.containing_type,
unittest_pb2.TestAllTypes.DESCRIPTOR)
unittest_pb2.TestAllTypes.DESCRIPTOR,
)
self.assertEqual(
unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.containing_type,
unittest_pb2.TestAllTypes.DESCRIPTOR)
unittest_pb2.TestAllTypes.DESCRIPTOR,
)
self.assertEqual(
unittest_pb2.TestAllTypes.RepeatedGroup.DESCRIPTOR.containing_type,
unittest_pb2.TestAllTypes.DESCRIPTOR)
unittest_pb2.TestAllTypes.DESCRIPTOR,
)
def testContainingTypeInEnumDescriptor(self):
self.assertTrue(unittest_pb2._FOREIGNENUM.containing_type is None)
self.assertEqual(unittest_pb2._TESTALLTYPES_NESTEDENUM.containing_type,
unittest_pb2.TestAllTypes.DESCRIPTOR)
self.assertEqual(
unittest_pb2._TESTALLTYPES_NESTEDENUM.containing_type,
unittest_pb2.TestAllTypes.DESCRIPTOR,
)
def testPackage(self):
self.assertEqual(
unittest_pb2.TestAllTypes.DESCRIPTOR.file.package,
'proto2_unittest')
unittest_pb2.TestAllTypes.DESCRIPTOR.file.package, 'proto2_unittest'
)
desc = unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR
self.assertEqual(desc.file.package, 'proto2_unittest')
self.assertEqual(
unittest_import_pb2.ImportMessage.DESCRIPTOR.file.package,
'proto2_unittest_import')
'proto2_unittest_import',
)
self.assertEqual(unittest_pb2._FOREIGNENUM.file.package, 'proto2_unittest')
self.assertEqual(
unittest_pb2._FOREIGNENUM.file.package, 'proto2_unittest')
unittest_pb2._TESTALLTYPES_NESTEDENUM.file.package, 'proto2_unittest'
)
self.assertEqual(
unittest_pb2._TESTALLTYPES_NESTEDENUM.file.package,
'proto2_unittest')
self.assertEqual(
unittest_import_pb2._IMPORTENUM.file.package,
'proto2_unittest_import')
unittest_import_pb2._IMPORTENUM.file.package, 'proto2_unittest_import'
)
def testExtensionRange(self):
self.assertEqual(
unittest_pb2.TestAllTypes.DESCRIPTOR.extension_ranges, [])
self.assertEqual(unittest_pb2.TestAllTypes.DESCRIPTOR.extension_ranges, [])
self.assertEqual(
unittest_pb2.TestAllExtensions.DESCRIPTOR.extension_ranges,
[(1, MAX_EXTENSION)])
[(1, MAX_EXTENSION)],
)
self.assertEqual(
unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR.extension_ranges,
[(42, 43), (4143, 4244), (65536, MAX_EXTENSION)])
[(42, 43), (4143, 4244), (65536, MAX_EXTENSION)],
)
def testFileDescriptor(self):
self.assertEqual(unittest_pb2.DESCRIPTOR.name,
'google/protobuf/unittest.proto')
self.assertEqual(
unittest_pb2.DESCRIPTOR.name, 'google/protobuf/unittest.proto'
)
self.assertEqual(unittest_pb2.DESCRIPTOR.package, 'proto2_unittest')
self.assertFalse(unittest_pb2.DESCRIPTOR.serialized_pb is None)
self.assertEqual(unittest_pb2.DESCRIPTOR.dependencies,
[unittest_import_pb2.DESCRIPTOR])
self.assertEqual(unittest_import_pb2.DESCRIPTOR.dependencies,
[unittest_import_public_pb2.DESCRIPTOR])
self.assertEqual(unittest_import_pb2.DESCRIPTOR.public_dependencies,
[unittest_import_public_pb2.DESCRIPTOR])
self.assertEqual(
unittest_pb2.DESCRIPTOR.dependencies, [unittest_import_pb2.DESCRIPTOR]
)
self.assertEqual(
unittest_import_pb2.DESCRIPTOR.dependencies,
[unittest_import_public_pb2.DESCRIPTOR],
)
self.assertEqual(
unittest_import_pb2.DESCRIPTOR.public_dependencies,
[unittest_import_public_pb2.DESCRIPTOR],
)
def testNoGenericServices(self):
self.assertTrue(hasattr(unittest_no_generic_services_pb2, "TestMessage"))
self.assertTrue(hasattr(unittest_no_generic_services_pb2, "FOO"))
self.assertTrue(hasattr(unittest_no_generic_services_pb2, "test_extension"))
self.assertTrue(hasattr(unittest_no_generic_services_pb2, 'TestMessage'))
self.assertTrue(hasattr(unittest_no_generic_services_pb2, 'FOO'))
self.assertTrue(hasattr(unittest_no_generic_services_pb2, 'test_extension'))
def testMessageTypesByName(self):
file_type = unittest_pb2.DESCRIPTOR
self.assertEqual(
unittest_pb2._TESTALLTYPES,
file_type.message_types_by_name[unittest_pb2._TESTALLTYPES.name])
file_type.message_types_by_name[unittest_pb2._TESTALLTYPES.name],
)
# Nested messages shouldn't be included in the message_types_by_name
# dictionary (like in the C++ API).
self.assertFalse(
unittest_pb2._TESTALLTYPES_NESTEDMESSAGE.name in
file_type.message_types_by_name)
unittest_pb2._TESTALLTYPES_NESTEDMESSAGE.name
in file_type.message_types_by_name
)
def testEnumTypesByName(self):
file_type = unittest_pb2.DESCRIPTOR
self.assertEqual(
unittest_pb2._FOREIGNENUM,
file_type.enum_types_by_name[unittest_pb2._FOREIGNENUM.name])
file_type.enum_types_by_name[unittest_pb2._FOREIGNENUM.name],
)
def testExtensionsByName(self):
file_type = unittest_pb2.DESCRIPTOR
self.assertEqual(
unittest_pb2.my_extension_string,
file_type.extensions_by_name[unittest_pb2.my_extension_string.name])
file_type.extensions_by_name[unittest_pb2.my_extension_string.name],
)
def testPublicImports(self):
# Test public imports as embedded message.
@ -315,20 +348,26 @@ class GeneratorTest(unittest.TestCase):
# module, and is public imported by unittest_import_pb2 module.
public_import_proto = unittest_import_pb2.PublicImportMessage()
self.assertEqual(0, public_import_proto.e)
self.assertTrue(unittest_import_public_pb2.PublicImportMessage is
unittest_import_pb2.PublicImportMessage)
self.assertTrue(
unittest_import_public_pb2.PublicImportMessage
is unittest_import_pb2.PublicImportMessage
)
def testBadIdentifiers(self):
# We're just testing that the code was imported without problems.
message = test_bad_identifiers_pb2.TestBadIdentifiers()
self.assertEqual(message.Extensions[test_bad_identifiers_pb2.message],
"foo")
self.assertEqual(message.Extensions[test_bad_identifiers_pb2.descriptor],
"bar")
self.assertEqual(message.Extensions[test_bad_identifiers_pb2.reflection],
"baz")
self.assertEqual(message.Extensions[test_bad_identifiers_pb2.service],
"qux")
self.assertEqual(
message.Extensions[test_bad_identifiers_pb2.message], 'foo'
)
self.assertEqual(
message.Extensions[test_bad_identifiers_pb2.descriptor], 'bar'
)
self.assertEqual(
message.Extensions[test_bad_identifiers_pb2.reflection], 'baz'
)
self.assertEqual(
message.Extensions[test_bad_identifiers_pb2.service], 'qux'
)
def testOneof(self):
desc = unittest_pb2.TestAllTypes.DESCRIPTOR
@ -347,8 +386,8 @@ class GeneratorTest(unittest.TestCase):
'oneof_lazy_nested_message',
])
self.assertEqual(
nested_names,
set([field.name for field in desc.oneofs[0].fields]))
nested_names, set([field.name for field in desc.oneofs[0].fields])
)
for field_name, field_desc in desc.fields_by_name.items():
if field_name in nested_names:
self.assertIs(desc.oneofs[0], field_desc.containing_oneof)
@ -356,14 +395,18 @@ class GeneratorTest(unittest.TestCase):
self.assertIsNone(field_desc.containing_oneof)
def testEnumWithDupValue(self):
self.assertEqual('FOO1',
unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.FOO1))
self.assertEqual('FOO1',
unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.FOO2))
self.assertEqual('BAR1',
unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.BAR1))
self.assertEqual('BAR1',
unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.BAR2))
self.assertEqual(
'FOO1', unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.FOO1)
)
self.assertEqual(
'FOO1', unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.FOO2)
)
self.assertEqual(
'BAR1', unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.BAR1)
)
self.assertEqual(
'BAR1', unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.BAR2)
)
class SymbolDatabaseRegistrationTest(unittest.TestCase):
@ -371,38 +414,52 @@ class SymbolDatabaseRegistrationTest(unittest.TestCase):
def testGetSymbol(self):
self.assertEqual(
unittest_pb2.TestAllTypes, symbol_database.Default().GetSymbol(
'proto2_unittest.TestAllTypes'))
unittest_pb2.TestAllTypes,
symbol_database.Default().GetSymbol('proto2_unittest.TestAllTypes'),
)
self.assertEqual(
unittest_pb2.TestAllTypes.NestedMessage,
symbol_database.Default().GetSymbol(
'proto2_unittest.TestAllTypes.NestedMessage'))
'proto2_unittest.TestAllTypes.NestedMessage'
),
)
with self.assertRaises(KeyError):
symbol_database.Default().GetSymbol('proto2_unittest.NestedMessage')
self.assertEqual(
unittest_pb2.TestAllTypes.OptionalGroup,
symbol_database.Default().GetSymbol(
'proto2_unittest.TestAllTypes.OptionalGroup'))
'proto2_unittest.TestAllTypes.OptionalGroup'
),
)
self.assertEqual(
unittest_pb2.TestAllTypes.RepeatedGroup,
symbol_database.Default().GetSymbol(
'proto2_unittest.TestAllTypes.RepeatedGroup'))
'proto2_unittest.TestAllTypes.RepeatedGroup'
),
)
def testEnums(self):
self.assertEqual(
'proto2_unittest.ForeignEnum',
symbol_database.Default().pool.FindEnumTypeByName(
'proto2_unittest.ForeignEnum').full_name)
symbol_database.Default()
.pool.FindEnumTypeByName('proto2_unittest.ForeignEnum')
.full_name,
)
self.assertEqual(
'proto2_unittest.TestAllTypes.NestedEnum',
symbol_database.Default().pool.FindEnumTypeByName(
'proto2_unittest.TestAllTypes.NestedEnum').full_name)
symbol_database.Default()
.pool.FindEnumTypeByName('proto2_unittest.TestAllTypes.NestedEnum')
.full_name,
)
def testFindFileByName(self):
self.assertEqual(
'google/protobuf/unittest.proto',
symbol_database.Default().pool.FindFileByName(
'google/protobuf/unittest.proto').name)
symbol_database.Default()
.pool.FindFileByName('google/protobuf/unittest.proto')
.name,
)
if __name__ == '__main__':
unittest.main()

3
python/google/protobuf/internal/keywords_test.py
View file

@ -9,9 +9,8 @@
import unittest
from google.protobuf.internal import more_messages_pb2
from google.protobuf import descriptor_pool
from google.protobuf.internal import more_messages_pb2
class KeywordsConflictTest(unittest.TestCase):

178
python/google/protobuf/internal/message_factory_test.py
View file

@ -9,27 +9,30 @@
__author__ = 'matthewtoia@google.com (Matt Toia)'
import unittest
import gc
import unittest
from google.protobuf import descriptor
from google.protobuf import descriptor_database
from google.protobuf import descriptor_pb2
from google.protobuf import descriptor_pool
from google.protobuf import message_factory
from google.protobuf.internal import api_implementation
from google.protobuf.internal import factory_test1_pb2
from google.protobuf.internal import factory_test2_pb2
from google.protobuf.internal import testing_refleaks
from google.protobuf import descriptor_database
from google.protobuf import descriptor_pool
from google.protobuf import message_factory
from google.protobuf import descriptor
@testing_refleaks.TestCase
class MessageFactoryTest(unittest.TestCase):
def setUp(self):
self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
factory_test1_pb2.DESCRIPTOR.serialized_pb)
factory_test1_pb2.DESCRIPTOR.serialized_pb
)
self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
factory_test2_pb2.DESCRIPTOR.serialized_pb)
factory_test2_pb2.DESCRIPTOR.serialized_pb
)
def _ExerciseDynamicClass(self, cls):
msg = cls()
@ -39,10 +42,11 @@ class MessageFactoryTest(unittest.TestCase):
msg.factory_1_message.factory_1_enum = 1
msg.factory_1_message.nested_factory_1_enum = 0
msg.factory_1_message.nested_factory_1_message.value = (
'nested message value')
'nested message value'
)
msg.factory_1_message.scalar_value = 22
msg.factory_1_message.list_value.extend([u'one', u'two', u'three'])
msg.factory_1_message.list_value.append(u'four')
msg.factory_1_message.list_value.extend(['one', 'two', 'three'])
msg.factory_1_message.list_value.append('four')
msg.factory_1_enum = 1
msg.nested_factory_1_enum = 0
msg.nested_factory_1_message.value = 'nested message value'
@ -50,8 +54,8 @@ class MessageFactoryTest(unittest.TestCase):
msg.circular_message.circular_message.mandatory = 2
msg.circular_message.scalar_value = 'one deep'
msg.scalar_value = 'zero deep'
msg.list_value.extend([u'four', u'three', u'two'])
msg.list_value.append(u'one')
msg.list_value.extend(['four', 'three', 'two'])
msg.list_value.append('one')
msg.grouped.add()
msg.grouped[0].part_1 = 'hello'
msg.grouped[0].part_2 = 'world'
@ -70,12 +74,18 @@ class MessageFactoryTest(unittest.TestCase):
pool = descriptor_pool.DescriptorPool(db)
db.Add(self.factory_test1_fd)
db.Add(self.factory_test2_fd)
cls = message_factory.GetMessageClass(pool.FindMessageTypeByName(
'google.protobuf.python.internal.Factory2Message'))
cls = message_factory.GetMessageClass(
pool.FindMessageTypeByName(
'google.protobuf.python.internal.Factory2Message'
)
)
self.assertFalse(cls is factory_test2_pb2.Factory2Message)
self._ExerciseDynamicClass(cls)
cls2 = message_factory.GetMessageClass(pool.FindMessageTypeByName(
'google.protobuf.python.internal.Factory2Message'))
cls2 = message_factory.GetMessageClass(
pool.FindMessageTypeByName(
'google.protobuf.python.internal.Factory2Message'
)
)
self.assertTrue(cls is cls2)
def testGetMessageClassWrongInput(self):
@ -95,11 +105,13 @@ class MessageFactoryTest(unittest.TestCase):
def testGetExistingPrototype(self):
# Get Existing Prototype should not create a new class.
cls = message_factory.GetMessageClass(
descriptor=factory_test2_pb2.Factory2Message.DESCRIPTOR)
descriptor=factory_test2_pb2.Factory2Message.DESCRIPTOR
)
msg = factory_test2_pb2.Factory2Message()
self.assertIsInstance(msg, cls)
self.assertIsInstance(msg.factory_1_message,
factory_test1_pb2.Factory1Message)
self.assertIsInstance(
msg.factory_1_message, factory_test1_pb2.Factory1Message
)
def testGetMessages(self):
# performed twice because multiple calls with the same input must be allowed
@ -109,44 +121,61 @@ class MessageFactoryTest(unittest.TestCase):
# are not in the topological order of dependencies.
# Assuming factory_test2_fd depends on factory_test1_fd.
self.assertIn(self.factory_test1_fd.name,
self.factory_test2_fd.dependency)
self.assertIn(
self.factory_test1_fd.name, self.factory_test2_fd.dependency
)
# Get messages should work when a file comes before its dependencies:
# factory_test2_fd comes before factory_test1_fd.
messages = message_factory.GetMessages([self.factory_test2_fd,
self.factory_test1_fd],
descriptor_pool.Default())
messages = message_factory.GetMessages(
[self.factory_test2_fd, self.factory_test1_fd],
descriptor_pool.Default(),
)
self.assertTrue(
set(['google.protobuf.python.internal.Factory2Message',
'google.protobuf.python.internal.Factory1Message'],
).issubset(set(messages.keys())))
set(
[
'google.protobuf.python.internal.Factory2Message',
'google.protobuf.python.internal.Factory1Message',
],
).issubset(set(messages.keys()))
)
self._ExerciseDynamicClass(
messages['google.protobuf.python.internal.Factory2Message'])
messages['google.protobuf.python.internal.Factory2Message']
)
factory_msg1 = messages['google.protobuf.python.internal.Factory1Message']
self.assertTrue(set(
['google.protobuf.python.internal.Factory2Message.one_more_field',
'google.protobuf.python.internal.another_field'],).issubset(set(
ext.full_name
for ext in factory_msg1.DESCRIPTOR.file.pool.FindAllExtensions(
factory_msg1.DESCRIPTOR))))
self.assertTrue(
set(
[
'google.protobuf.python.internal.Factory2Message.one_more_field',
'google.protobuf.python.internal.another_field',
],
).issubset(
set(
ext.full_name
for ext in (
factory_msg1.DESCRIPTOR.file.pool.FindAllExtensions(
factory_msg1.DESCRIPTOR
)
)
)
)
)
msg1 = messages['google.protobuf.python.internal.Factory1Message']()
ext1 = msg1.Extensions._FindExtensionByName(
'google.protobuf.python.internal.Factory2Message.one_more_field')
'google.protobuf.python.internal.Factory2Message.one_more_field'
)
ext2 = msg1.Extensions._FindExtensionByName(
'google.protobuf.python.internal.another_field')
'google.protobuf.python.internal.another_field'
)
self.assertEqual(0, len(msg1.Extensions))
msg1.Extensions[ext1] = 'test1'
msg1.Extensions[ext2] = 'test2'
self.assertEqual('test1', msg1.Extensions[ext1])
self.assertEqual('test2', msg1.Extensions[ext2])
self.assertEqual(None,
msg1.Extensions._FindExtensionByNumber(12321))
self.assertEqual(None, msg1.Extensions._FindExtensionByNumber(12321))
self.assertEqual(2, len(msg1.Extensions))
if api_implementation.Type() == 'python':
self.assertEqual(None,
msg1.Extensions._FindExtensionByName(0))
self.assertEqual(None,
msg1.Extensions._FindExtensionByNumber(''))
self.assertEqual(None, msg1.Extensions._FindExtensionByName(0))
self.assertEqual(None, msg1.Extensions._FindExtensionByNumber(''))
else:
self.assertRaises(TypeError, msg1.Extensions._FindExtensionByName, 0)
self.assertRaises(TypeError, msg1.Extensions._FindExtensionByNumber, '')
@ -157,7 +186,8 @@ class MessageFactoryTest(unittest.TestCase):
# Add Container message.
f = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/container.proto',
package='google.protobuf.python.internal')
package='google.protobuf.python.internal',
)
f.message_type.add(name='Container').extension_range.add(start=1, end=10)
pool.Add(f)
msgs = message_factory.GetMessageClassesForFiles([f.name], pool)
@ -167,7 +197,8 @@ class MessageFactoryTest(unittest.TestCase):
f = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/extension.proto',
package='google.protobuf.python.internal',
dependency=['google/protobuf/internal/container.proto'])
dependency=['google/protobuf/internal/container.proto'],
)
msg = f.message_type.add(name='Extension')
msg.extension.add(
name='extension_field',
@ -184,7 +215,8 @@ class MessageFactoryTest(unittest.TestCase):
f = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/duplicate.proto',
package='google.protobuf.python.internal',
dependency=['google/protobuf/internal/container.proto'])
dependency=['google/protobuf/internal/container.proto'],
)
msg = f.message_type.add(name='Duplicate')
msg.extension.add(
name='extension_field',
@ -231,25 +263,29 @@ class MessageFactoryTest(unittest.TestCase):
# Add Container message.
f1 = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/container.proto',
package='google.protobuf.python.internal')
package='google.protobuf.python.internal',
)
f1.message_type.add(name='Container').extension_range.add(start=1, end=10)
# Add ValueType message.
f2 = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/value_type.proto',
package='google.protobuf.python.internal')
package='google.protobuf.python.internal',
)
f2.message_type.add(name='ValueType').field.add(
name='setting',
number=1,
label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL,
type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32,
default_value='123')
default_value='123',
)
# Extend container with field of ValueType.
f3 = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/extension.proto',
package='google.protobuf.python.internal',
dependency=[f1.name, f2.name])
dependency=[f1.name, f2.name],
)
f3.extension.add(
name='top_level_extension_field',
number=2,
@ -273,7 +309,8 @@ class MessageFactoryTest(unittest.TestCase):
except:
pass
msgs = message_factory.GetMessageClassesForFiles(
[f1.name, f3.name], pool) # Deliberately not f2.
[f1.name, f3.name], pool
) # Deliberately not f2.
msg = msgs['google.protobuf.python.internal.Container']
desc = msgs['google.protobuf.python.internal.Extension'].DESCRIPTOR
ext1 = desc.file.extensions_by_name['top_level_extension_field']
@ -283,12 +320,12 @@ class MessageFactoryTest(unittest.TestCase):
m.Extensions[ext2].setting = 345
serialized = m.SerializeToString()
f1.name='google/protobuf/internal/another/container.proto'
f1.package='google.protobuf.python.internal.another'
f2.name='google/protobuf/internal/another/value_type.proto'
f2.package='google.protobuf.python.internal.another'
f3.name='google/protobuf/internal/another/extension.proto'
f3.package='google.protobuf.python.internal.another'
f1.name = 'google/protobuf/internal/another/container.proto'
f1.package = 'google.protobuf.python.internal.another'
f2.name = 'google/protobuf/internal/another/value_type.proto'
f2.package = 'google.protobuf.python.internal.another'
f3.name = 'google/protobuf/internal/another/extension.proto'
f3.package = 'google.protobuf.python.internal.another'
f3.ClearField('dependency')
f3.dependency.extend([f1.name, f2.name])
try:
@ -298,7 +335,8 @@ class MessageFactoryTest(unittest.TestCase):
except:
pass
msgs = message_factory.GetMessageClassesForFiles(
[f1.name, f3.name], pool) # Deliberately not f2.
[f1.name, f3.name], pool
) # Deliberately not f2.
msg = msgs['google.protobuf.python.internal.another.Container']
desc = msgs['google.protobuf.python.internal.another.Extension'].DESCRIPTOR
ext1 = desc.file.extensions_by_name['top_level_extension_field']
@ -310,16 +348,19 @@ class MessageFactoryTest(unittest.TestCase):
def testDescriptorKeepConcreteClass(self):
def loadFile():
f= descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/meta_class.proto',
package='google.protobuf.python.internal')
f = descriptor_pb2.FileDescriptorProto(
name='google/protobuf/internal/meta_class.proto',
package='google.protobuf.python.internal',
)
msg_proto = f.message_type.add(name='Empty')
msg_proto.nested_type.add(name='Nested')
msg_proto.field.add(name='nested_field',
number=1,
label=descriptor.FieldDescriptor.LABEL_REPEATED,
type=descriptor.FieldDescriptor.TYPE_MESSAGE,
type_name='Nested')
msg_proto.field.add(
name='nested_field',
number=1,
label=descriptor.FieldDescriptor.LABEL_REPEATED,
type=descriptor.FieldDescriptor.TYPE_MESSAGE,
type_name='Nested',
)
return message_factory.GetMessages([f])
messages = loadFile()
@ -331,7 +372,8 @@ class MessageFactoryTest(unittest.TestCase):
def testOndemandCreateMetaClass(self):
def loadFile():
f = descriptor_pb2.FileDescriptorProto.FromString(
factory_test1_pb2.DESCRIPTOR.serialized_pb)
factory_test1_pb2.DESCRIPTOR.serialized_pb
)
return message_factory.GetMessages([f])
messages = loadFile()
@ -345,12 +387,14 @@ class MessageFactoryTest(unittest.TestCase):
value = message.map_field
values = [
# The entry class will be created on demand in upb python.
value.GetEntryClass()(key=k, value=value[k]) for k in sorted(value)
value.GetEntryClass()(key=k, value=value[k])
for k in sorted(value)
]
gc.collect()
self.assertEqual(1, len(values))
self.assertEqual('hello', values[0].key)
self.assertEqual('welcome', values[0].value)
if __name__ == '__main__':
unittest.main()

11
python/google/protobuf/internal/message_listener.py
View file

@ -6,6 +6,7 @@
# https://developers.google.com/open-source/licenses/bsd
"""Defines a listener interface for observing certain
state transitions on Message objects.
Also defines a null implementation of this interface.
@ -15,17 +16,18 @@ __author__ = 'robinson@google.com (Will Robinson)'
class MessageListener(object):
"""Listens for modifications made to a message.
"""Listens for modifications made to a message. Meant to be registered via
Message._SetListener().
Meant to be registered via Message._SetListener().
Attributes:
dirty: If True, then calling Modified() would be a no-op. This can be
used to avoid these calls entirely in the common case.
dirty: If True, then calling Modified() would be a no-op. This can be used
to avoid these calls entirely in the common case.
"""
def Modified(self):
"""Called every time the message is modified in such a way that the parent
message may need to be updated. This currently means either:
(a) The message was modified for the first time, so the parent message
should henceforth mark the message as present.
@ -48,7 +50,6 @@ class MessageListener(object):
class NullMessageListener(object):
"""No-op MessageListener implementation."""
def Modified(self):

499
python/google/protobuf/internal/message_test.py
View file

File diff suppressed because it is too large Load diff

1
python/google/protobuf/internal/numpy/__init__.py
View file

@ -4,4 +4,3 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd

6
python/google/protobuf/internal/numpy/numpy_test.py
View file

@ -189,9 +189,9 @@ class NumpyProtoIndexingTest(unittest.TestCase):
with self.assertRaises(TypeError):
_ = data.repeated_int64[np.ndarray((1,), buffer=np.array([0]), dtype=int)]
with self.assertRaises(TypeError):
_ = data.repeated_int64[np.ndarray((1, 1),
buffer=np.array([0]),
dtype=int)]
_ = data.repeated_int64[
np.ndarray((1, 1), buffer=np.array([0]), dtype=int)
]
@testing_refleaks.TestCase

20
python/google/protobuf/internal/proto_builder_test.py
View file

@ -26,30 +26,34 @@ class ProtoBuilderTest(unittest.TestCase):
self.ordered_fields = collections.OrderedDict([
('foo', descriptor_pb2.FieldDescriptorProto.TYPE_INT64),
('bar', descriptor_pb2.FieldDescriptorProto.TYPE_STRING),
])
])
self._fields = dict(self.ordered_fields)
def testMakeSimpleProtoClass(self):
"""Test that we can create a proto class."""
proto_cls = proto_builder.MakeSimpleProtoClass(
self._fields,
full_name='net.proto2.python.public.proto_builder_test.Test')
full_name='net.proto2.python.public.proto_builder_test.Test',
)
proto = proto_cls()
proto.foo = 12345
proto.bar = 'asdf'
self.assertMultiLineEqual(
'bar: "asdf"\nfoo: 12345\n', text_format.MessageToString(proto))
'bar: "asdf"\nfoo: 12345\n', text_format.MessageToString(proto)
)
def testOrderedFields(self):
"""Test that the field order is maintained when given an OrderedDict."""
proto_cls = proto_builder.MakeSimpleProtoClass(
self.ordered_fields,
full_name='net.proto2.python.public.proto_builder_test.OrderedTest')
full_name='net.proto2.python.public.proto_builder_test.OrderedTest',
)
proto = proto_cls()
proto.foo = 12345
proto.bar = 'asdf'
self.assertMultiLineEqual(
'foo: 12345\nbar: "asdf"\n', text_format.MessageToString(proto))
'foo: 12345\nbar: "asdf"\n', text_format.MessageToString(proto)
)
def testMakeSameProtoClassTwice(self):
"""Test that the DescriptorPool is used."""
@ -57,11 +61,13 @@ class ProtoBuilderTest(unittest.TestCase):
proto_cls1 = proto_builder.MakeSimpleProtoClass(
self._fields,
full_name='net.proto2.python.public.proto_builder_test.Test',
pool=pool)
pool=pool,
)
proto_cls2 = proto_builder.MakeSimpleProtoClass(
self._fields,
full_name='net.proto2.python.public.proto_builder_test.Test',
pool=pool)
pool=pool,
)
self.assertIs(proto_cls1.DESCRIPTOR, proto_cls2.DESCRIPTOR)
def testMakeLargeProtoClass(self):

6
python/google/protobuf/internal/proto_json_test.py
View file

@ -11,6 +11,7 @@
import unittest
from google.protobuf import proto_json
from google.protobuf.util import json_format_proto3_pb2
@ -25,10 +26,9 @@ class ProtoJsonTest(unittest.TestCase):
def test_simple_parse(self):
expected = 12345
js_dict = {'int32Value': expected}
message = proto_json.parse(json_format_proto3_pb2.TestMessage,
js_dict)
message = proto_json.parse(json_format_proto3_pb2.TestMessage, js_dict)
self.assertEqual(expected, message.int32_value) # pytype: disable=attribute-error
if __name__ == "__main__":
if __name__ == '__main__':
unittest.main()

397
python/google/protobuf/internal/python_message.py
View file

@ -9,8 +9,8 @@
#
# TODO: Helpers for verbose, common checks like seeing if a
# descriptor's cpp_type is CPPTYPE_MESSAGE.
"""Contains a metaclass and helper functions used to create
protocol message classes from Descriptor objects at runtime.
Recall that a metaclass is the "type" of a class.
@ -38,6 +38,7 @@ import weakref
from google.protobuf import descriptor as descriptor_mod
from google.protobuf import message as message_mod
from google.protobuf import text_format
# We use "as" to avoid name collisions with variables.
from google.protobuf.internal import api_implementation
from google.protobuf.internal import containers
@ -56,8 +57,8 @@ _StructFullTypeName = 'google.protobuf.Struct'
_ListValueFullTypeName = 'google.protobuf.ListValue'
_ExtensionDict = extension_dict._ExtensionDict
class GeneratedProtocolMessageType(type):
class GeneratedProtocolMessageType(type):
"""Metaclass for protocol message classes created at runtime from Descriptors.
We add implementations for all methods described in the Message class. We
@ -91,15 +92,13 @@ class GeneratedProtocolMessageType(type):
(The interplay between metaclasses and slots is not very well-documented).
Args:
name: Name of the class (ignored, but required by the
metaclass protocol).
bases: Base classes of the class we're constructing.
(Should be message.Message). We ignore this field, but
it's required by the metaclass protocol
dictionary: The class dictionary of the class we're
constructing. dictionary[_DESCRIPTOR_KEY] must contain
a Descriptor object describing this protocol message
type.
name: Name of the class (ignored, but required by the metaclass protocol).
bases: Base classes of the class we're constructing. (Should be
message.Message). We ignore this field, but it's required by the
metaclass protocol
dictionary: The class dictionary of the class we're constructing.
dictionary[_DESCRIPTOR_KEY] must contain a Descriptor object describing
this protocol message type.
Returns:
Newly-allocated class.
@ -110,8 +109,10 @@ class GeneratedProtocolMessageType(type):
descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY]
if isinstance(descriptor, str):
raise RuntimeError('The generated code only work with python cpp '
'extension, but it is using pure python runtime.')
raise RuntimeError(
'The generated code only work with python cpp '
'extension, but it is using pure python runtime.'
)
# If a concrete class already exists for this descriptor, don't try to
# create another. Doing so will break any messages that already exist with
@ -138,20 +139,18 @@ class GeneratedProtocolMessageType(type):
def __init__(cls, name, bases, dictionary):
"""Here we perform the majority of our work on the class.
We add enum getters, an __init__ method, implementations
of all Message methods, and properties for all fields
in the protocol type.
We add enum getters, an __init__ method, implementations of all Message
methods, and properties for all fields in the protocol type.
Args:
name: Name of the class (ignored, but required by the
metaclass protocol).
bases: Base classes of the class we're constructing.
(Should be message.Message). We ignore this field, but
it's required by the metaclass protocol
dictionary: The class dictionary of the class we're
constructing. dictionary[_DESCRIPTOR_KEY] must contain
a Descriptor object describing this protocol message
type.
name: Name of the class (ignored, but required by the metaclass protocol).
bases: Base classes of the class we're constructing. (Should be
message.Message). We ignore this field, but it's required by the
metaclass protocol
dictionary: The class dictionary of the class we're constructing.
dictionary[_DESCRIPTOR_KEY] must contain a Descriptor object describing
this protocol message type.
"""
descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY]
@ -161,13 +160,16 @@ class GeneratedProtocolMessageType(type):
if existing_class:
assert existing_class is cls, (
'Duplicate `GeneratedProtocolMessageType` created for descriptor %r'
% (descriptor.full_name))
% (descriptor.full_name)
)
return
cls._message_set_decoders_by_tag = {}
cls._fields_by_tag = {}
if (descriptor.has_options and
descriptor.GetOptions().message_set_wire_format):
if (
descriptor.has_options
and descriptor.GetOptions().message_set_wire_format
):
cls._message_set_decoders_by_tag[decoder.MESSAGE_SET_ITEM_TAG] = (
decoder.MessageSetItemDecoder(descriptor),
None,
@ -205,12 +207,13 @@ class GeneratedProtocolMessageType(type):
def _PropertyName(proto_field_name):
"""Returns the name of the public property attribute which
clients can use to get and (in some cases) set the value
of a protocol message field.
Args:
proto_field_name: The protocol message field name, exactly
as it appears (or would appear) in a .proto file.
proto_field_name: The protocol message field name, exactly as it appears (or
would appear) in a .proto file.
"""
# TODO: Escape Python keywords (e.g., yield), and test this support.
# nnorwitz makes my day by writing:
@ -234,41 +237,49 @@ def _PropertyName(proto_field_name):
def _AddSlots(message_descriptor, dictionary):
"""Adds a __slots__ entry to dictionary, containing the names of all valid
attributes for this message type.
Args:
message_descriptor: A Descriptor instance describing this message type.
dictionary: Class dictionary to which we'll add a '__slots__' entry.
"""
dictionary['__slots__'] = ['_cached_byte_size',
'_cached_byte_size_dirty',
'_fields',
'_unknown_fields',
'_is_present_in_parent',
'_listener',
'_listener_for_children',
'__weakref__',
'_oneofs']
dictionary['__slots__'] = [
'_cached_byte_size',
'_cached_byte_size_dirty',
'_fields',
'_unknown_fields',
'_is_present_in_parent',
'_listener',
'_listener_for_children',
'__weakref__',
'_oneofs',
]
def _IsMessageSetExtension(field):
return (field.is_extension and
field.containing_type.has_options and
field.containing_type.GetOptions().message_set_wire_format and
field.type == _FieldDescriptor.TYPE_MESSAGE and
not field.is_required and
not field.is_repeated)
return (
field.is_extension
and field.containing_type.has_options
and field.containing_type.GetOptions().message_set_wire_format
and field.type == _FieldDescriptor.TYPE_MESSAGE
and not field.is_required
and not field.is_repeated
)
def _IsMapField(field):
return (field.type == _FieldDescriptor.TYPE_MESSAGE and
field.message_type._is_map_entry)
return (
field.type == _FieldDescriptor.TYPE_MESSAGE
and field.message_type._is_map_entry
)
def _IsMessageMapField(field):
value_type = field.message_type.fields_by_name['value']
return value_type.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE
def _AttachFieldHelpers(cls, field_descriptor):
field_descriptor._default_constructor = _DefaultValueConstructorForField(
field_descriptor
@ -299,16 +310,19 @@ def _MaybeAddEncoder(cls, field_descriptor):
if is_map_entry:
field_encoder = encoder.MapEncoder(field_descriptor)
sizer = encoder.MapSizer(field_descriptor,
_IsMessageMapField(field_descriptor))
sizer = encoder.MapSizer(
field_descriptor, _IsMessageMapField(field_descriptor)
)
elif _IsMessageSetExtension(field_descriptor):
field_encoder = encoder.MessageSetItemEncoder(field_descriptor.number)
sizer = encoder.MessageSetItemSizer(field_descriptor.number)
else:
field_encoder = type_checkers.TYPE_TO_ENCODER[field_descriptor.type](
field_descriptor.number, is_repeated, is_packed)
field_descriptor.number, is_repeated, is_packed
)
sizer = type_checkers.TYPE_TO_SIZER[field_descriptor.type](
field_descriptor.number, is_repeated, is_packed)
field_descriptor.number, is_repeated, is_packed
)
field_descriptor._sizer = sizer
field_descriptor._encoder = field_encoder
@ -324,8 +338,10 @@ def _MaybeAddDecoder(cls, field_descriptor):
def AddDecoder(is_packed):
decode_type = field_descriptor.type
if (decode_type == _FieldDescriptor.TYPE_ENUM and
not field_descriptor.enum_type.is_closed):
if (
decode_type == _FieldDescriptor.TYPE_ENUM
and not field_descriptor.enum_type.is_closed
):
decode_type = _FieldDescriptor.TYPE_INT32
oneof_descriptor = None
@ -336,23 +352,37 @@ def _MaybeAddDecoder(cls, field_descriptor):
is_message_map = _IsMessageMapField(field_descriptor)
field_decoder = decoder.MapDecoder(
field_descriptor, _GetInitializeDefaultForMap(field_descriptor),
is_message_map)
field_descriptor,
_GetInitializeDefaultForMap(field_descriptor),
is_message_map,
)
elif decode_type == _FieldDescriptor.TYPE_STRING:
field_decoder = decoder.StringDecoder(
field_descriptor.number, is_repeated, is_packed,
field_descriptor, field_descriptor._default_constructor,
not field_descriptor.has_presence)
field_descriptor.number,
is_repeated,
is_packed,
field_descriptor,
field_descriptor._default_constructor,
not field_descriptor.has_presence,
)
elif field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
field_descriptor.number, is_repeated, is_packed,
field_descriptor, field_descriptor._default_constructor)
field_descriptor.number,
is_repeated,
is_packed,
field_descriptor,
field_descriptor._default_constructor,
)
else:
field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
field_descriptor.number, is_repeated, is_packed,
field_descriptor.number,
is_repeated,
is_packed,
# pylint: disable=protected-access
field_descriptor, field_descriptor._default_constructor,
not field_descriptor.has_presence)
field_descriptor,
field_descriptor._default_constructor,
not field_descriptor.has_presence,
)
helper_decoders[is_packed] = field_decoder
@ -390,26 +420,36 @@ def _AddEnumValues(descriptor, cls):
def _GetInitializeDefaultForMap(field):
if not field.is_repeated:
raise ValueError('map_entry set on non-repeated field %s' % (
field.name))
raise ValueError('map_entry set on non-repeated field %s' % (field.name))
fields_by_name = field.message_type.fields_by_name
key_checker = type_checkers.GetTypeChecker(fields_by_name['key'])
value_field = fields_by_name['value']
if _IsMessageMapField(field):
def MakeMessageMapDefault(message):
return containers.MessageMap(
message._listener_for_children, value_field.message_type, key_checker,
field.message_type)
message._listener_for_children,
value_field.message_type,
key_checker,
field.message_type,
)
return MakeMessageMapDefault
else:
value_checker = type_checkers.GetTypeChecker(value_field)
def MakePrimitiveMapDefault(message):
return containers.ScalarMap(
message._listener_for_children, key_checker, value_checker,
field.message_type)
message._listener_for_children,
key_checker,
value_checker,
field.message_type,
)
return MakePrimitiveMapDefault
def _DefaultValueConstructorForField(field):
"""Returns a function which returns a default value for a field.
@ -429,43 +469,55 @@ def _DefaultValueConstructorForField(field):
if field.is_repeated:
if field.has_default_value and field.default_value != []:
raise ValueError('Repeated field default value not empty list: %s' % (
field.default_value))
raise ValueError(
'Repeated field default value not empty list: %s'
% (field.default_value)
)
if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
# We can't look at _concrete_class yet since it might not have
# been set. (Depends on order in which we initialize the classes).
message_type = field.message_type
def MakeRepeatedMessageDefault(message):
return containers.RepeatedCompositeFieldContainer(
message._listener_for_children, field.message_type)
message._listener_for_children, field.message_type
)
return MakeRepeatedMessageDefault
else:
type_checker = type_checkers.GetTypeChecker(field)
def MakeRepeatedScalarDefault(message):
return containers.RepeatedScalarFieldContainer(
message._listener_for_children, type_checker, field
)
return MakeRepeatedScalarDefault
if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
message_type = field.message_type
def MakeSubMessageDefault(message):
# _concrete_class may not yet be initialized.
if not hasattr(message_type, '_concrete_class'):
from google.protobuf import message_factory
message_factory.GetMessageClass(message_type)
result = message_type._concrete_class()
result._SetListener(
_OneofListener(message, field)
if field.containing_oneof is not None
else message._listener_for_children)
else message._listener_for_children
)
return result
return MakeSubMessageDefault
def MakeScalarDefault(message):
# TODO: This may be broken since there may not be
# default_value. Combine with has_default_value somehow.
return field.default_value
return MakeScalarDefault
@ -494,8 +546,9 @@ def _AddInitMethod(message_descriptor, cls):
try:
return enum_type.values_by_name[value].number
except KeyError:
raise ValueError('Enum type %s: unknown label "%s"' % (
enum_type.full_name, value))
raise ValueError(
'Enum type %s: unknown label "%s"' % (enum_type.full_name, value)
)
return value
def init(self, **kwargs):
@ -544,8 +597,10 @@ def _AddInitMethod(message_descriptor, cls):
for field_name, field_value in kwargs.items():
field = _GetFieldByName(message_descriptor, field_name)
if field is None:
raise TypeError('%s() got an unexpected keyword argument "%s"' %
(message_descriptor.name, field_name))
raise TypeError(
'%s() got an unexpected keyword argument "%s"'
% (message_descriptor.name, field_name)
)
if field_value is None:
# field=None is the same as no field at all.
continue
@ -573,8 +628,10 @@ def _AddInitMethod(message_descriptor, cls):
init_wkt_or_merge(field, new_msg, val)
else: # Scalar
if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
field_value = [_GetIntegerEnumValue(field.enum_type, val)
for val in field_value]
field_value = [
_GetIntegerEnumValue(field.enum_type, val)
for val in field_value
]
field_copy.extend(field_value)
self._fields[field] = field_copy
elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
@ -609,14 +666,17 @@ def _GetFieldByName(message_descriptor, field_name):
Args:
message_descriptor: A Descriptor describing all fields in message.
field_name: The name of the field to retrieve.
Returns:
The field descriptor associated with the field name.
"""
try:
return message_descriptor.fields_by_name[field_name]
except KeyError:
raise ValueError('Protocol message %s has no "%s" field.' %
(message_descriptor.name, field_name))
raise ValueError(
'Protocol message %s has no "%s" field.'
% (message_descriptor.name, field_name)
)
def _AddPropertiesForFields(descriptor, cls):
@ -632,9 +692,9 @@ def _AddPropertiesForFields(descriptor, cls):
def _AddPropertiesForField(field, cls):
"""Adds a public property for a protocol message field.
Clients can use this property to get and (in the case
of non-repeated scalar fields) directly set the value
of a protocol message field.
Clients can use this property to get and (in the case of non-repeated scalar
fields) directly set the value of a protocol message field.
Args:
field: A FieldDescriptor for this field.
@ -664,9 +724,10 @@ class _FieldProperty(property):
def _AddPropertiesForRepeatedField(field, cls):
"""Adds a public property for a "repeated" protocol message field. Clients
can use this property to get the value of the field, which will be either a
RepeatedScalarFieldContainer or RepeatedCompositeFieldContainer (see
"""Adds a public property for a "repeated" protocol message field.
Clients can use this property to get the value of the field, which will be
either a RepeatedScalarFieldContainer or RepeatedCompositeFieldContainer (see
below).
Note that when clients add values to these containers, we perform
@ -694,14 +755,17 @@ def _AddPropertiesForRepeatedField(field, cls):
# in several other locations in this file.
field_value = self._fields.setdefault(field, field_value)
return field_value
getter.__module__ = None
getter.__doc__ = 'Getter for %s.' % proto_field_name
# We define a setter just so we can throw an exception with a more
# helpful error message.
def setter(self, new_value):
raise AttributeError('Assignment not allowed to repeated field '
'"%s" in protocol message object.' % proto_field_name)
raise AttributeError(
'Assignment not allowed to repeated field '
'"%s" in protocol message object.' % proto_field_name
)
doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name
setattr(cls, property_name, _FieldProperty(field, getter, setter, doc=doc))
@ -709,6 +773,7 @@ def _AddPropertiesForRepeatedField(field, cls):
def _AddPropertiesForNonRepeatedScalarField(field, cls):
"""Adds a public property for a nonrepeated, scalar protocol message field.
Clients can use this property to get and directly set the value of the field.
Note that when the client sets the value of a field by using this property,
all necessary "has" bits are set as a side-effect, and we also perform
@ -727,6 +792,7 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
# TODO: This may be broken since there may not be
# default_value. Combine with has_default_value somehow.
return self._fields.get(field, default_value)
getter.__module__ = None
getter.__doc__ = 'Getter for %s.' % proto_field_name
@ -738,7 +804,8 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
new_value = type_checker.CheckValue(new_value)
except TypeError as e:
raise TypeError(
'Cannot set %s to %.1024r: %s' % (field.full_name, new_value, e))
'Cannot set %s to %.1024r: %s' % (field.full_name, new_value, e)
)
if not field.has_presence and decoder.IsDefaultScalarValue(new_value):
self._fields.pop(field, None)
else:
@ -749,9 +816,11 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
self._Modified()
if field.containing_oneof:
def setter(self, new_value):
field_setter(self, new_value)
self._UpdateOneofState(field)
else:
setter = field_setter
@ -765,6 +834,7 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
def _AddPropertiesForNonRepeatedCompositeField(field, cls):
"""Adds a public property for a nonrepeated, composite protocol message field.
A composite field is a "group" or "message" field.
Clients can use this property to get the value of the field, but cannot
@ -793,6 +863,7 @@ def _AddPropertiesForNonRepeatedCompositeField(field, cls):
# in several other locations in this file.
field_value = self._fields.setdefault(field, field_value)
return field_value
getter.__module__ = None
getter.__doc__ = 'Getter for %s.' % proto_field_name
@ -837,6 +908,7 @@ def _AddPropertiesForExtensions(descriptor, cls):
# TODO: Use cls.MESSAGE_FACTORY.pool when available.
pool = descriptor.file.pool
def _AddStaticMethods(cls):
def RegisterExtension(_):
@ -845,16 +917,20 @@ def _AddStaticMethods(cls):
pass
cls.RegisterExtension = staticmethod(RegisterExtension)
def FromString(s):
message = cls()
message.MergeFromString(s)
return message
cls.FromString = staticmethod(FromString)
def _IsPresent(item):
"""Given a (FieldDescriptor, value) tuple from _fields, return true if the
value should be included in the list returned by ListFields()."""
value should be included in the list returned by ListFields().
"""
if item[0].is_repeated:
return bool(item[1])
@ -869,7 +945,7 @@ def _AddListFieldsMethod(message_descriptor, cls):
def ListFields(self):
all_fields = [item for item in self._fields.items() if _IsPresent(item)]
all_fields.sort(key = lambda item: item[0].number)
all_fields.sort(key=lambda item: item[0].number)
return all_fields
cls.ListFields = ListFields
@ -895,9 +971,11 @@ def _AddHasFieldMethod(message_descriptor, cls):
try:
field = hassable_fields[field_name]
except KeyError as exc:
raise ValueError('Protocol message %s has no non-repeated field "%s" '
'nor has presence is not available for this field.' % (
message_descriptor.full_name, field_name)) from exc
raise ValueError(
'Protocol message %s has no non-repeated field "%s" '
'nor has presence is not available for this field.'
% (message_descriptor.full_name, field_name)
) from exc
if isinstance(field, descriptor_mod.OneofDescriptor):
try:
@ -916,6 +994,7 @@ def _AddHasFieldMethod(message_descriptor, cls):
def _AddClearFieldMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def ClearField(self, field_name):
try:
field = message_descriptor.fields_by_name[field_name]
@ -927,8 +1006,10 @@ def _AddClearFieldMethod(message_descriptor, cls):
else:
return
except KeyError:
raise ValueError('Protocol message %s has no "%s" field.' %
(message_descriptor.name, field_name))
raise ValueError(
'Protocol message %s has no "%s" field.'
% (message_descriptor.name, field_name)
)
if field in self._fields:
# To match the C++ implementation, we need to invalidate iterators
@ -954,6 +1035,7 @@ def _AddClearFieldMethod(message_descriptor, cls):
def _AddClearExtensionMethod(cls):
"""Helper for _AddMessageMethods()."""
def ClearExtension(self, field_descriptor):
extension_dict._VerifyExtensionHandle(self, field_descriptor)
@ -961,11 +1043,13 @@ def _AddClearExtensionMethod(cls):
if field_descriptor in self._fields:
del self._fields[field_descriptor]
self._Modified()
cls.ClearExtension = ClearExtension
def _AddHasExtensionMethod(cls):
"""Helper for _AddMessageMethods()."""
def HasExtension(self, field_descriptor):
extension_dict._VerifyExtensionHandle(self, field_descriptor)
if field_descriptor.is_repeated:
@ -976,8 +1060,10 @@ def _AddHasExtensionMethod(cls):
return value is not None and value._is_present_in_parent
else:
return field_descriptor in self._fields
cls.HasExtension = HasExtension
def _InternalUnpackAny(msg):
"""Unpacks Any message and returns the unpacked message.
@ -996,6 +1082,7 @@ def _InternalUnpackAny(msg):
# parent message.
# pylint: disable=g-import-not-at-top
from google.protobuf import symbol_database
factory = symbol_database.Default()
type_url = msg.type_url
@ -1014,6 +1101,7 @@ def _InternalUnpackAny(msg):
# Unable to import message_factory at top because of circular import.
# pylint: disable=g-import-not-at-top
from google.protobuf import message_factory
message_class = message_factory.GetMessageClass(descriptor)
message = message_class()
@ -1023,6 +1111,7 @@ def _InternalUnpackAny(msg):
def _AddEqualsMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def __eq__(self, other):
if self.DESCRIPTOR.full_name == _ListValueFullTypeName and isinstance(
other, list
@ -1033,8 +1122,10 @@ def _AddEqualsMethod(message_descriptor, cls):
):
return self._internal_compare(other)
if (not isinstance(other, message_mod.Message) or
other.DESCRIPTOR != self.DESCRIPTOR):
if (
not isinstance(other, message_mod.Message)
or other.DESCRIPTOR != self.DESCRIPTOR
):
return NotImplemented
if self is other:
@ -1062,15 +1153,19 @@ def _AddEqualsMethod(message_descriptor, cls):
def _AddStrMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def __str__(self):
return text_format.MessageToString(self)
cls.__str__ = __str__
def _AddReprMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def __repr__(self):
return text_format.MessageToString(self)
cls.__repr__ = __repr__
@ -1079,18 +1174,24 @@ def _AddUnicodeMethod(unused_message_descriptor, cls):
def __unicode__(self):
return text_format.MessageToString(self, as_utf8=True).decode('utf-8')
cls.__unicode__ = __unicode__
def _AddContainsMethod(message_descriptor, cls):
if message_descriptor.full_name == 'google.protobuf.Struct':
def __contains__(self, key):
return key in self.fields
elif message_descriptor.full_name == 'google.protobuf.ListValue':
def __contains__(self, value):
return value in self.items()
else:
def __contains__(self, field):
return self.HasField(field)
@ -1099,16 +1200,17 @@ def _AddContainsMethod(message_descriptor, cls):
def _BytesForNonRepeatedElement(value, field_number, field_type):
"""Returns the number of bytes needed to serialize a non-repeated element.
The returned byte count includes space for tag information and any
other additional space associated with serializing value.
The returned byte count includes space for tag information and any other
additional space associated with serializing value.
Args:
value: Value we're serializing.
field_number: Field number of this value. (Since the field number
is stored as part of a varint-encoded tag, this has an impact
on the total bytes required to serialize the value).
field_type: The type of the field. One of the TYPE_* constants
within FieldDescriptor.
field_number: Field number of this value. (Since the field number is stored
as part of a varint-encoded tag, this has an impact on the total bytes
required to serialize the value).
field_type: The type of the field. One of the TYPE_* constants within
FieldDescriptor.
"""
try:
fn = type_checkers.TYPE_TO_BYTE_SIZE_FN[field_type]
@ -1156,9 +1258,14 @@ def _AddSerializeToStringMethod(message_descriptor, cls):
# Check if the message has all of its required fields set.
if not self.IsInitialized():
raise message_mod.EncodeError(
'Message %s is missing required fields: %s' % (
self.DESCRIPTOR.full_name, ','.join(self.FindInitializationErrors())))
'Message %s is missing required fields: %s'
% (
self.DESCRIPTOR.full_name,
','.join(self.FindInitializationErrors()),
)
)
return self.SerializePartialToString(**kwargs)
cls.SerializeToString = SerializeToString
@ -1169,12 +1276,14 @@ def _AddSerializePartialToStringMethod(message_descriptor, cls):
out = BytesIO()
self._InternalSerialize(out.write, **kwargs)
return out.getvalue()
cls.SerializePartialToString = SerializePartialToString
def InternalSerialize(self, write_bytes, deterministic=None):
if deterministic is None:
deterministic = (
api_implementation.IsPythonDefaultSerializationDeterministic())
api_implementation.IsPythonDefaultSerializationDeterministic()
)
else:
deterministic = bool(deterministic)
@ -1194,11 +1303,13 @@ def _AddSerializePartialToStringMethod(message_descriptor, cls):
for tag_bytes, value_bytes in self._unknown_fields:
write_bytes(tag_bytes)
write_bytes(value_bytes)
cls._InternalSerialize = InternalSerialize
def _AddMergeFromStringMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def MergeFromString(self, serialized):
serialized = memoryview(serialized)
length = len(serialized)
@ -1212,7 +1323,8 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
raise message_mod.DecodeError('Truncated message.')
except struct.error as e:
raise message_mod.DecodeError(e)
return length # Return this for legacy reasons.
return length # Return this for legacy reasons.
cls.MergeFromString = MergeFromString
fields_by_tag = cls._fields_by_tag
@ -1236,7 +1348,7 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
self._Modified()
field_dict = self._fields
while pos != end:
(tag_bytes, new_pos) = decoder.ReadTag(buffer, pos)
tag_bytes, new_pos = decoder.ReadTag(buffer, pos)
field_decoder, field_des = message_set_decoders_by_tag.get(
tag_bytes, (None, None)
)
@ -1247,12 +1359,12 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
continue
field_des, is_packed = fields_by_tag.get(tag_bytes, (None, None))
if field_des is None:
if not self._unknown_fields: # pylint: disable=protected-access
self._unknown_fields = [] # pylint: disable=protected-access
if not self._unknown_fields: # pylint: disable=protected-access
self._unknown_fields = [] # pylint: disable=protected-access
field_number, wire_type = decoder.DecodeTag(tag_bytes)
if field_number == 0:
raise message_mod.DecodeError('Field number 0 is illegal.')
(data, new_pos) = decoder._DecodeUnknownField(
data, new_pos = decoder._DecodeUnknownField(
buffer, new_pos, end, field_number, wire_type
) # pylint: disable=protected-access
if new_pos == -1:
@ -1276,17 +1388,20 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
def _AddIsInitializedMethod(message_descriptor, cls):
"""Adds the IsInitialized and FindInitializationError methods to the
protocol message class."""
required_fields = [field for field in message_descriptor.fields
if field.is_required]
protocol message class.
"""
required_fields = [
field for field in message_descriptor.fields if field.is_required
]
def IsInitialized(self, errors=None):
"""Checks if all required fields of a message are set.
Args:
errors: A list which, if provided, will be populated with the field
paths of all missing required fields.
errors: A list which, if provided, will be populated with the field paths
of all missing required fields.
Returns:
True iff the specified message has all required fields set.
@ -1295,9 +1410,10 @@ def _AddIsInitializedMethod(message_descriptor, cls):
# Performance is critical so we avoid HasField() and ListFields().
for field in required_fields:
if (field not in self._fields or
(field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE and
not self._fields[field]._is_present_in_parent)):
if field not in self._fields or (
field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE
and not self._fields[field]._is_present_in_parent
):
if errors is not None:
errors.extend(self.FindInitializationErrors())
return False
@ -1305,7 +1421,7 @@ def _AddIsInitializedMethod(message_descriptor, cls):
for field, value in list(self._fields.items()): # dict can change size!
if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
if field.is_repeated:
if (field.message_type._is_map_entry):
if field.message_type._is_map_entry:
continue
for element in value:
if not element.IsInitialized():
@ -1383,8 +1499,12 @@ def _AddMergeFromMethod(cls):
if not isinstance(msg, cls):
raise TypeError(
'Parameter to MergeFrom() must be instance of same class: '
'expected %s got %s.' % (_FullyQualifiedClassName(cls),
_FullyQualifiedClassName(msg.__class__)))
'expected %s got %s.'
% (
_FullyQualifiedClassName(cls),
_FullyQualifiedClassName(msg.__class__),
)
)
assert msg is not self
self._Modified()
@ -1421,13 +1541,13 @@ def _AddMergeFromMethod(cls):
def _AddWhichOneofMethod(message_descriptor, cls):
def WhichOneof(self, oneof_name):
"""Returns the name of the currently set field inside a oneof, or None."""
try:
field = message_descriptor.oneofs_by_name[oneof_name]
except KeyError:
raise ValueError(
'Protocol message has no oneof "%s" field.' % oneof_name)
raise ValueError('Protocol message has no oneof "%s" field.' % oneof_name)
nested_field = self._oneofs.get(field, None)
if nested_field is not None and self.HasField(nested_field.name):
@ -1448,9 +1568,11 @@ def _Clear(self):
def _UnknownFields(self):
raise NotImplementedError('Please use the add-on feaure '
'unknown_fields.UnknownFieldSet(message) in '
'unknown_fields.py instead.')
raise NotImplementedError(
'Please use the add-on feaure '
'unknown_fields.UnknownFieldSet(message) in '
'unknown_fields.py instead.'
)
def _DiscardUnknownFields(self):
@ -1506,6 +1628,7 @@ def _AddPrivateHelperMethods(message_descriptor, cls):
def Modified(self):
"""Sets the _cached_byte_size_dirty bit to true,
and propagates this to our listener iff this was a state change.
"""
@ -1536,8 +1659,8 @@ def _AddPrivateHelperMethods(message_descriptor, cls):
class _Listener(object):
"""MessageListener implementation that a parent message registers with its
child message.
In order to support semantics like:
@ -1551,8 +1674,9 @@ class _Listener(object):
def __init__(self, parent_message):
"""Args:
parent_message: The message whose _Modified() method we should call when
we receive Modified() messages.
parent_message: The message whose _Modified() method we should call when
we receive Modified() messages.
"""
# This listener establishes a back reference from a child (contained) object
# to its parent (containing) object. We make this a weak reference to avoid
@ -1586,9 +1710,10 @@ class _OneofListener(_Listener):
def __init__(self, parent_message, field):
"""Args:
parent_message: The message whose _Modified() method we should call when
we receive Modified() messages.
field: The descriptor of the field being set in the parent message.
parent_message: The message whose _Modified() method we should call when
we receive Modified() messages.
field: The descriptor of the field being set in the parent message.
"""
super(_OneofListener, self).__init__(parent_message)
self._field = field

1
python/google/protobuf/internal/reflection_cpp_test.py
View file

@ -12,6 +12,7 @@ import copy
import unittest
from google.protobuf.internal import testing_refleaks
from absl.testing import parameterized
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2

1097
python/google/protobuf/internal/reflection_test.py
View file

File diff suppressed because it is too large Load diff

1
python/google/protobuf/internal/runtime_version_test.py
View file

@ -10,6 +10,7 @@
__author__ = 'shaod@google.com (Dennis Shao)'
import unittest
from google.protobuf import runtime_version

54
python/google/protobuf/internal/service_reflection_test.py
View file

@ -13,6 +13,7 @@ __author__ = 'petar@google.com (Petar Petrov)'
import unittest
from google.protobuf import service_reflection
from google.protobuf import unittest_pb2
@ -21,6 +22,7 @@ class FooUnitTest(unittest.TestCase):
def testService(self):
class MockRpcChannel:
def CallMethod(self, method, controller, request, response, callback):
self.method = method
self.controller = controller
@ -28,6 +30,7 @@ class FooUnitTest(unittest.TestCase):
callback(response)
class MockRpcController:
def SetFailed(self, msg):
self.failure_message = msg
@ -45,26 +48,38 @@ class FooUnitTest(unittest.TestCase):
channel = MockRpcChannel()
srvc = MyService()
srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback)
self.assertEqual('Method Foo not implemented.',
rpc_controller.failure_message)
self.assertEqual(
'Method Foo not implemented.', rpc_controller.failure_message
)
self.assertEqual(None, self.callback_response)
rpc_controller.failure_message = None
service_descriptor = unittest_pb2.TestService.GetDescriptor()
srvc.CallMethod(service_descriptor.methods[1], rpc_controller,
unittest_pb2.BarRequest(), MyCallback)
self.assertTrue(srvc.GetRequestClass(service_descriptor.methods[1]) is
unittest_pb2.BarRequest)
self.assertTrue(srvc.GetResponseClass(service_descriptor.methods[1]) is
unittest_pb2.BarResponse)
self.assertEqual('Method Bar not implemented.',
rpc_controller.failure_message)
srvc.CallMethod(
service_descriptor.methods[1],
rpc_controller,
unittest_pb2.BarRequest(),
MyCallback,
)
self.assertTrue(
srvc.GetRequestClass(service_descriptor.methods[1])
is unittest_pb2.BarRequest
)
self.assertTrue(
srvc.GetResponseClass(service_descriptor.methods[1])
is unittest_pb2.BarResponse
)
self.assertEqual(
'Method Bar not implemented.', rpc_controller.failure_message
)
self.assertEqual(None, self.callback_response)
class MyServiceImpl(unittest_pb2.TestService):
def Foo(self, rpc_controller, request, done):
self.foo_called = True
def Bar(self, rpc_controller, request, done):
self.bar_called = True
@ -75,16 +90,22 @@ class FooUnitTest(unittest.TestCase):
self.assertEqual(True, srvc.foo_called)
rpc_controller.failure_message = None
srvc.CallMethod(service_descriptor.methods[1], rpc_controller,
unittest_pb2.BarRequest(), MyCallback)
srvc.CallMethod(
service_descriptor.methods[1],
rpc_controller,
unittest_pb2.BarRequest(),
MyCallback,
)
self.assertEqual(None, rpc_controller.failure_message)
self.assertEqual(True, srvc.bar_called)
def testServiceStub(self):
class MockRpcChannel:
def CallMethod(self, method, controller, request,
response_class, callback):
def CallMethod(
self, method, controller, request, response_class, callback
):
self.method = method
self.controller = controller
self.request = request
@ -101,8 +122,9 @@ class FooUnitTest(unittest.TestCase):
request = 'request'
# GetDescriptor now static, still works as instance method for compatibility
self.assertEqual(unittest_pb2.TestService_Stub.GetDescriptor(),
stub.GetDescriptor())
self.assertEqual(
unittest_pb2.TestService_Stub.GetDescriptor(), stub.GetDescriptor()
)
# Invoke method.
stub.Foo(rpc_controller, request, MyCallback)

79
python/google/protobuf/internal/symbol_database_test.py
View file

@ -12,6 +12,7 @@ import unittest
from google.protobuf import descriptor
from google.protobuf import descriptor_pool
from google.protobuf import symbol_database
from google.protobuf import unittest_pb2
@ -37,61 +38,91 @@ class SymbolDatabaseTest(unittest.TestCase):
def testGetSymbol(self):
self.assertEqual(
unittest_pb2.TestAllTypes, self._Database().GetSymbol(
'proto2_unittest.TestAllTypes'))
unittest_pb2.TestAllTypes,
self._Database().GetSymbol('proto2_unittest.TestAllTypes'),
)
self.assertEqual(
unittest_pb2.TestAllTypes.NestedMessage, self._Database().GetSymbol(
'proto2_unittest.TestAllTypes.NestedMessage'))
unittest_pb2.TestAllTypes.NestedMessage,
self._Database().GetSymbol(
'proto2_unittest.TestAllTypes.NestedMessage'
),
)
self.assertEqual(
unittest_pb2.TestAllTypes.OptionalGroup, self._Database().GetSymbol(
'proto2_unittest.TestAllTypes.OptionalGroup'))
unittest_pb2.TestAllTypes.OptionalGroup,
self._Database().GetSymbol(
'proto2_unittest.TestAllTypes.OptionalGroup'
),
)
self.assertEqual(
unittest_pb2.TestAllTypes.RepeatedGroup, self._Database().GetSymbol(
'proto2_unittest.TestAllTypes.RepeatedGroup'))
unittest_pb2.TestAllTypes.RepeatedGroup,
self._Database().GetSymbol(
'proto2_unittest.TestAllTypes.RepeatedGroup'
),
)
def testEnums(self):
# Check registration of types in the pool.
self.assertEqual(
'proto2_unittest.ForeignEnum',
self._Database().pool.FindEnumTypeByName(
'proto2_unittest.ForeignEnum').full_name)
self._Database()
.pool.FindEnumTypeByName('proto2_unittest.ForeignEnum')
.full_name,
)
self.assertEqual(
'proto2_unittest.TestAllTypes.NestedEnum',
self._Database().pool.FindEnumTypeByName(
'proto2_unittest.TestAllTypes.NestedEnum').full_name)
self._Database()
.pool.FindEnumTypeByName('proto2_unittest.TestAllTypes.NestedEnum')
.full_name,
)
def testFindMessageTypeByName(self):
self.assertEqual(
'proto2_unittest.TestAllTypes',
self._Database().pool.FindMessageTypeByName(
'proto2_unittest.TestAllTypes').full_name)
self._Database()
.pool.FindMessageTypeByName('proto2_unittest.TestAllTypes')
.full_name,
)
self.assertEqual(
'proto2_unittest.TestAllTypes.NestedMessage',
self._Database().pool.FindMessageTypeByName(
'proto2_unittest.TestAllTypes.NestedMessage').full_name)
self._Database()
.pool.FindMessageTypeByName(
'proto2_unittest.TestAllTypes.NestedMessage'
)
.full_name,
)
def testFindServiceByName(self):
self.assertEqual(
'proto2_unittest.TestService',
self._Database().pool.FindServiceByName(
'proto2_unittest.TestService').full_name)
self._Database()
.pool.FindServiceByName('proto2_unittest.TestService')
.full_name,
)
def testFindFileContainingSymbol(self):
# Lookup based on either enum or message.
self.assertEqual(
'google/protobuf/unittest.proto',
self._Database().pool.FindFileContainingSymbol(
'proto2_unittest.TestAllTypes.NestedEnum').name)
self._Database()
.pool.FindFileContainingSymbol(
'proto2_unittest.TestAllTypes.NestedEnum'
)
.name,
)
self.assertEqual(
'google/protobuf/unittest.proto',
self._Database().pool.FindFileContainingSymbol(
'proto2_unittest.TestAllTypes').name)
self._Database()
.pool.FindFileContainingSymbol('proto2_unittest.TestAllTypes')
.name,
)
def testFindFileByName(self):
self.assertEqual(
'google/protobuf/unittest.proto',
self._Database().pool.FindFileByName(
'google/protobuf/unittest.proto').name)
self._Database()
.pool.FindFileByName('google/protobuf/unittest.proto')
.name,
)
if __name__ == '__main__':

147
python/google/protobuf/internal/test_util.py
View file

@ -23,7 +23,7 @@ from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2
try:
long # Python 2
long # Python 2
except NameError:
long = int # Python 3
@ -47,21 +47,21 @@ def SetAllNonLazyFields(message):
# Optional fields.
#
message.optional_int32 = 101
message.optional_int64 = 102
message.optional_uint32 = 103
message.optional_uint64 = 104
message.optional_sint32 = 105
message.optional_sint64 = 106
message.optional_fixed32 = 107
message.optional_fixed64 = 108
message.optional_int32 = 101
message.optional_int64 = 102
message.optional_uint32 = 103
message.optional_uint64 = 104
message.optional_sint32 = 105
message.optional_sint64 = 106
message.optional_fixed32 = 107
message.optional_fixed64 = 108
message.optional_sfixed32 = 109
message.optional_sfixed64 = 110
message.optional_float = 111
message.optional_double = 112
message.optional_bool = True
message.optional_string = u'115'
message.optional_bytes = b'116'
message.optional_float = 111
message.optional_double = 112
message.optional_bool = True
message.optional_string = '115'
message.optional_bytes = b'116'
if IsProto2(message):
message.optionalgroup.a = 117
@ -79,8 +79,8 @@ def SetAllNonLazyFields(message):
if IsProto2(message):
message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ
message.optional_string_piece = u'124'
message.optional_cord = u'125'
message.optional_string_piece = '124'
message.optional_cord = '125'
if hasattr(message, 'optional_bytes_cord'):
message.optional_bytes_cord = b'optional bytes cord'
@ -101,7 +101,7 @@ def SetAllNonLazyFields(message):
message.repeated_float.append(211)
message.repeated_double.append(212)
message.repeated_bool.append(True)
message.repeated_string.append(u'215')
message.repeated_string.append('215')
message.repeated_bytes.append(b'216')
if IsProto2(message):
@ -117,8 +117,8 @@ def SetAllNonLazyFields(message):
if IsProto2(message):
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR)
message.repeated_string_piece.append(u'224')
message.repeated_cord.append(u'225')
message.repeated_string_piece.append('224')
message.repeated_cord.append('225')
# Add a second one of each field and set value by index.
message.repeated_int32.append(0)
@ -134,7 +134,7 @@ def SetAllNonLazyFields(message):
message.repeated_float.append(0)
message.repeated_double.append(0)
message.repeated_bool.append(True)
message.repeated_string.append(u'0')
message.repeated_string.append('0')
message.repeated_bytes.append(b'0')
message.repeated_int32[1] = 301
message.repeated_int64[1] = 302
@ -149,7 +149,7 @@ def SetAllNonLazyFields(message):
message.repeated_float[1] = 311
message.repeated_double[1] = 312
message.repeated_bool[1] = False
message.repeated_string[1] = u'315'
message.repeated_string[1] = '315'
message.repeated_bytes[1] = b'316'
if IsProto2(message):
@ -166,8 +166,8 @@ def SetAllNonLazyFields(message):
if IsProto2(message):
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)
message.repeated_string_piece.append(u'324')
message.repeated_cord.append(u'325')
message.repeated_string_piece.append('324')
message.repeated_cord.append('325')
#
# Fields that have defaults.
@ -239,7 +239,7 @@ def SetAllExtensions(message):
extensions[pb2.optional_float_extension] = 111
extensions[pb2.optional_double_extension] = 112
extensions[pb2.optional_bool_extension] = True
extensions[pb2.optional_string_extension] = u'115'
extensions[pb2.optional_string_extension] = '115'
extensions[pb2.optional_bytes_extension] = b'116'
extensions[pb2.optionalgroup_extension].a = 117
@ -255,8 +255,8 @@ def SetAllExtensions(message):
extensions[pb2.optional_foreign_enum_extension] = pb2.FOREIGN_BAZ
extensions[pb2.optional_import_enum_extension] = import_pb2.IMPORT_BAZ
extensions[pb2.optional_string_piece_extension] = u'124'
extensions[pb2.optional_cord_extension] = u'125'
extensions[pb2.optional_string_piece_extension] = '124'
extensions[pb2.optional_cord_extension] = '125'
extensions[pb2.optional_bytes_cord_extension] = b'optional bytes cord'
#
@ -276,7 +276,7 @@ def SetAllExtensions(message):
extensions[pb2.repeated_float_extension].append(211)
extensions[pb2.repeated_double_extension].append(212)
extensions[pb2.repeated_bool_extension].append(True)
extensions[pb2.repeated_string_extension].append(u'215')
extensions[pb2.repeated_string_extension].append('215')
extensions[pb2.repeated_bytes_extension].append(b'216')
extensions[pb2.repeatedgroup_extension].add().a = 217
@ -289,8 +289,8 @@ def SetAllExtensions(message):
extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAR)
extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAR)
extensions[pb2.repeated_string_piece_extension].append(u'224')
extensions[pb2.repeated_cord_extension].append(u'225')
extensions[pb2.repeated_string_piece_extension].append('224')
extensions[pb2.repeated_cord_extension].append('225')
# Append a second one of each field.
extensions[pb2.repeated_int32_extension].append(301)
@ -306,7 +306,7 @@ def SetAllExtensions(message):
extensions[pb2.repeated_float_extension].append(311)
extensions[pb2.repeated_double_extension].append(312)
extensions[pb2.repeated_bool_extension].append(False)
extensions[pb2.repeated_string_extension].append(u'315')
extensions[pb2.repeated_string_extension].append('315')
extensions[pb2.repeated_bytes_extension].append(b'316')
extensions[pb2.repeatedgroup_extension].add().a = 317
@ -319,8 +319,8 @@ def SetAllExtensions(message):
extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAZ)
extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAZ)
extensions[pb2.repeated_string_piece_extension].append(u'324')
extensions[pb2.repeated_cord_extension].append(u'325')
extensions[pb2.repeated_string_piece_extension].append('324')
extensions[pb2.repeated_cord_extension].append('325')
#
# Fields with defaults.
@ -339,19 +339,19 @@ def SetAllExtensions(message):
extensions[pb2.default_float_extension] = 411
extensions[pb2.default_double_extension] = 412
extensions[pb2.default_bool_extension] = False
extensions[pb2.default_string_extension] = u'415'
extensions[pb2.default_string_extension] = '415'
extensions[pb2.default_bytes_extension] = b'416'
extensions[pb2.default_nested_enum_extension] = pb2.TestAllTypes.FOO
extensions[pb2.default_foreign_enum_extension] = pb2.FOREIGN_FOO
extensions[pb2.default_import_enum_extension] = import_pb2.IMPORT_FOO
extensions[pb2.default_string_piece_extension] = u'424'
extensions[pb2.default_string_piece_extension] = '424'
extensions[pb2.default_cord_extension] = '425'
extensions[pb2.oneof_uint32_extension] = 601
extensions[pb2.oneof_nested_message_extension].bb = 602
extensions[pb2.oneof_string_extension] = u'603'
extensions[pb2.oneof_string_extension] = '603'
extensions[pb2.oneof_bytes_extension] = b'604'
@ -370,6 +370,7 @@ def SetAllFieldsAndExtensions(message):
def ExpectAllFieldsAndExtensionsInOrder(serialized):
"""Ensures that serialized is the serialization we expect for a message
filled with SetAllFieldsAndExtensions(). (Specifically, ensures that the
serialization is in canonical, tag-number order).
"""
@ -461,13 +462,14 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(127, message.optional_lazy_message.bb)
test_case.assertEqual(128, message.optional_unverified_lazy_message.bb)
test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ,
message.optional_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
message.optional_foreign_enum)
test_case.assertEqual(
unittest_pb2.TestAllTypes.BAZ, message.optional_nested_enum
)
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, message.optional_foreign_enum)
if IsProto2(message):
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.optional_import_enum)
test_case.assertEqual(
unittest_import_pb2.IMPORT_BAZ, message.optional_import_enum
)
# -----------------------------------------------------------------
@ -523,13 +525,16 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(220, message.repeated_import_message[0].d)
test_case.assertEqual(227, message.repeated_lazy_message[0].bb)
test_case.assertEqual(unittest_pb2.TestAllTypes.BAR,
message.repeated_nested_enum[0])
test_case.assertEqual(unittest_pb2.FOREIGN_BAR,
message.repeated_foreign_enum[0])
test_case.assertEqual(
unittest_pb2.TestAllTypes.BAR, message.repeated_nested_enum[0]
)
test_case.assertEqual(
unittest_pb2.FOREIGN_BAR, message.repeated_foreign_enum[0]
)
if IsProto2(message):
test_case.assertEqual(unittest_import_pb2.IMPORT_BAR,
message.repeated_import_enum[0])
test_case.assertEqual(
unittest_import_pb2.IMPORT_BAR, message.repeated_import_enum[0]
)
test_case.assertEqual(301, message.repeated_int32[1])
test_case.assertEqual(302, message.repeated_int64[1])
@ -554,13 +559,16 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(320, message.repeated_import_message[1].d)
test_case.assertEqual(327, message.repeated_lazy_message[1].bb)
test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ,
message.repeated_nested_enum[1])
test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
message.repeated_foreign_enum[1])
test_case.assertEqual(
unittest_pb2.TestAllTypes.BAZ, message.repeated_nested_enum[1]
)
test_case.assertEqual(
unittest_pb2.FOREIGN_BAZ, message.repeated_foreign_enum[1]
)
if IsProto2(message):
test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
message.repeated_import_enum[1])
test_case.assertEqual(
unittest_import_pb2.IMPORT_BAZ, message.repeated_import_enum[1]
)
# -----------------------------------------------------------------
@ -601,12 +609,15 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual('415', message.default_string)
test_case.assertEqual(b'416', message.default_bytes)
test_case.assertEqual(unittest_pb2.TestAllTypes.FOO,
message.default_nested_enum)
test_case.assertEqual(unittest_pb2.FOREIGN_FOO,
message.default_foreign_enum)
test_case.assertEqual(unittest_import_pb2.IMPORT_FOO,
message.default_import_enum)
test_case.assertEqual(
unittest_pb2.TestAllTypes.FOO, message.default_nested_enum
)
test_case.assertEqual(
unittest_pb2.FOREIGN_FOO, message.default_foreign_enum
)
test_case.assertEqual(
unittest_import_pb2.IMPORT_FOO, message.default_import_enum
)
def GoldenFile(filename):
@ -640,7 +651,8 @@ def GoldenFile(filename):
raise RuntimeError(
'Could not find golden files. This test must be run from within the '
'protobuf source package so that it can read test data files from the '
'C++ source tree.')
'C++ source tree.'
)
def GoldenFileData(filename):
@ -668,8 +680,9 @@ def SetAllPackedFields(message):
message.packed_float.extend([611.0, 711.0])
message.packed_double.extend([612.0, 712.0])
message.packed_bool.extend([True, False])
message.packed_enum.extend([unittest_pb2.FOREIGN_BAR,
unittest_pb2.FOREIGN_BAZ])
message.packed_enum.extend(
[unittest_pb2.FOREIGN_BAR, unittest_pb2.FOREIGN_BAZ]
)
def SetAllPackedExtensions(message):
@ -694,8 +707,9 @@ def SetAllPackedExtensions(message):
extensions[pb2.packed_float_extension].extend([611.0, 711.0])
extensions[pb2.packed_double_extension].extend([612.0, 712.0])
extensions[pb2.packed_bool_extension].extend([True, False])
extensions[pb2.packed_enum_extension].extend([unittest_pb2.FOREIGN_BAR,
unittest_pb2.FOREIGN_BAZ])
extensions[pb2.packed_enum_extension].extend(
[unittest_pb2.FOREIGN_BAR, unittest_pb2.FOREIGN_BAZ]
)
def SetAllUnpackedFields(message):
@ -717,8 +731,9 @@ def SetAllUnpackedFields(message):
message.unpacked_float.extend([611.0, 711.0])
message.unpacked_double.extend([612.0, 712.0])
message.unpacked_bool.extend([True, False])
message.unpacked_enum.extend([unittest_pb2.FOREIGN_BAR,
unittest_pb2.FOREIGN_BAZ])
message.unpacked_enum.extend(
[unittest_pb2.FOREIGN_BAR, unittest_pb2.FOREIGN_BAZ]
)
class NonStandardInteger(numbers.Integral):

16
python/google/protobuf/internal/testing_refleaks.py
View file

@ -48,8 +48,12 @@ class ReferenceLeakCheckerMixin(object):
def run(self, result=None):
testMethod = getattr(self, self._testMethodName)
expecting_failure_method = getattr(testMethod, "__unittest_expecting_failure__", False)
expecting_failure_class = getattr(self, "__unittest_expecting_failure__", False)
expecting_failure_method = getattr(
testMethod, "__unittest_expecting_failure__", False
)
expecting_failure_class = getattr(
self, "__unittest_expecting_failure__", False
)
if expecting_failure_class or expecting_failure_method:
return
@ -106,13 +110,15 @@ class ReferenceLeakCheckerMixin(object):
return sys.gettotalrefcount()
if hasattr(sys, 'gettotalrefcount'):
if hasattr(sys, "gettotalrefcount"):
def TestCase(test_class):
new_bases = (ReferenceLeakCheckerMixin,) + test_class.__bases__
new_class = type(test_class)(
test_class.__name__, new_bases, dict(test_class.__dict__))
test_class.__name__, new_bases, dict(test_class.__dict__)
)
return new_class
SkipReferenceLeakChecker = unittest.skip
else:
@ -123,6 +129,8 @@ else:
def SkipReferenceLeakChecker(reason):
del reason # Don't skip, so don't need a reason.
def Same(func):
return func
return Same

30
python/google/protobuf/internal/text_encoding_test.py
View file

@ -12,21 +12,27 @@ import unittest
from google.protobuf import text_encoding
TEST_VALUES = [
("foo\\rbar\\nbaz\\t",
"foo\\rbar\\nbaz\\t",
b"foo\rbar\nbaz\t"),
("\\'full of \\\"sound\\\" and \\\"fury\\\"\\'",
"\\'full of \\\"sound\\\" and \\\"fury\\\"\\'",
b"'full of \"sound\" and \"fury\"'"),
("signi\\\\fying\\\\ nothing\\\\",
"signi\\\\fying\\\\ nothing\\\\",
b"signi\\fying\\ nothing\\"),
("\\010\\t\\n\\013\\014\\r",
"\\010\\t\\n\\013\\014\\r",
b"\010\011\012\013\014\015")]
("foo\\rbar\\nbaz\\t", "foo\\rbar\\nbaz\\t", b"foo\rbar\nbaz\t"),
(
'\\\'full of \\"sound\\" and \\"fury\\"\\\'',
'\\\'full of \\"sound\\" and \\"fury\\"\\\'',
b'\'full of "sound" and "fury"\'',
),
(
"signi\\\\fying\\\\ nothing\\\\",
"signi\\\\fying\\\\ nothing\\\\",
b"signi\\fying\\ nothing\\",
),
(
"\\010\\t\\n\\013\\014\\r",
"\\010\\t\\n\\013\\014\\r",
b"\010\011\012\013\014\015",
),
]
class TextEncodingTestCase(unittest.TestCase):
def testCEscape(self):
for escaped, escaped_utf8, unescaped in TEST_VALUES:
self.assertEqual(escaped, text_encoding.CEscape(unescaped, as_utf8=False))

1385
python/google/protobuf/internal/text_format_test.py
View file

File diff suppressed because it is too large Load diff

133
python/google/protobuf/internal/type_checkers.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Provides type checking routines.
This module defines type checking utilities in the forms of dictionaries:
@ -33,6 +32,7 @@ from google.protobuf.internal import wire_format
_FieldDescriptor = descriptor.FieldDescriptor
def TruncateToFourByteFloat(original):
return struct.unpack('<f', struct.pack('<f', original))[0]
@ -62,8 +62,10 @@ def GetTypeChecker(field):
An instance of TypeChecker which can be used to verify the types
of values assigned to a field of the specified type.
"""
if (field.cpp_type == _FieldDescriptor.CPPTYPE_STRING and
field.type == _FieldDescriptor.TYPE_STRING):
if (
field.cpp_type == _FieldDescriptor.CPPTYPE_STRING
and field.type == _FieldDescriptor.TYPE_STRING
):
return UnicodeValueChecker()
if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
if field.enum_type.is_closed:
@ -79,8 +81,8 @@ def GetTypeChecker(field):
# protect against malicious clients here, just people accidentally shooting
# themselves in the foot in obvious ways.
class TypeChecker(object):
"""Type checker used to catch type errors as early as possible
when the client is setting scalar fields in protocol messages.
"""
@ -93,8 +95,11 @@ class TypeChecker(object):
The returned value might have been normalized to another type.
"""
if not isinstance(proposed_value, self._acceptable_types):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), self._acceptable_types))
message = '%.1024r has type %s, but expected one of: %s' % (
proposed_value,
type(proposed_value),
self._acceptable_types,
)
raise TypeError(message)
return proposed_value
@ -115,17 +120,27 @@ class BoolValueChecker(object):
def CheckValue(self, proposed_value):
if not hasattr(proposed_value, '__index__'):
# Under NumPy 2.3, numpy.bool does not have an __index__ method.
if (type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'bool'):
if (
type(proposed_value).__module__ == 'numpy'
and type(proposed_value).__name__ == 'bool'
):
return bool(proposed_value)
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (bool, int)))
message = '%.1024r has type %s, but expected one of: %s' % (
proposed_value,
type(proposed_value),
(bool, int),
)
raise TypeError(message)
if (type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'ndarray'):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (bool, int)))
if (
type(proposed_value).__module__ == 'numpy'
and type(proposed_value).__name__ == 'ndarray'
):
message = '%.1024r has type %s, but expected one of: %s' % (
proposed_value,
type(proposed_value),
(bool, int),
)
raise TypeError(message)
return bool(proposed_value)
@ -137,7 +152,6 @@ class BoolValueChecker(object):
# IntValueChecker and its subclasses perform integer type-checks
# and bounds-checks.
class IntValueChecker(object):
"""Checker used for integer fields. Performs type-check and range check."""
def CheckValue(self, proposed_value):
@ -152,10 +166,14 @@ class IntValueChecker(object):
raise TypeError(message)
if not hasattr(proposed_value, '__index__') or (
type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'ndarray'):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (int,)))
type(proposed_value).__module__ == 'numpy'
and type(proposed_value).__name__ == 'ndarray'
):
message = '%.1024r has type %s, but expected one of: %s' % (
proposed_value,
type(proposed_value),
(int,),
)
raise TypeError(message)
if not self._MIN <= int(proposed_value) <= self._MAX:
@ -170,7 +188,6 @@ class IntValueChecker(object):
class EnumValueChecker(object):
"""Checker used for enum fields. Performs type-check and range check."""
def __init__(self, enum_type):
@ -188,8 +205,11 @@ class EnumValueChecker(object):
raise TypeError(message)
if not isinstance(proposed_value, numbers.Integral):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (int,)))
message = '%.1024r has type %s, but expected one of: %s' % (
proposed_value,
type(proposed_value),
(int,),
)
raise TypeError(message)
if int(proposed_value) not in self._enum_type.values_by_number:
raise ValueError('Unknown enum value: %d' % proposed_value)
@ -200,7 +220,6 @@ class EnumValueChecker(object):
class UnicodeValueChecker(object):
"""Checker used for string fields.
Always returns a unicode value, even if the input is of type str.
@ -208,8 +227,11 @@ class UnicodeValueChecker(object):
def CheckValue(self, proposed_value):
if not isinstance(proposed_value, (bytes, str)):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (bytes, str)))
message = '%.1024r has type %s, but expected one of: %s' % (
proposed_value,
type(proposed_value),
(bytes, str),
)
raise TypeError(message)
# If the value is of type 'bytes' make sure that it is valid UTF-8 data.
@ -217,22 +239,24 @@ class UnicodeValueChecker(object):
try:
proposed_value = proposed_value.decode('utf-8')
except UnicodeDecodeError:
raise ValueError('%.1024r has type bytes, but isn\'t valid UTF-8 '
'encoding. Non-UTF-8 strings must be converted to '
'unicode objects before being added.' %
(proposed_value))
raise ValueError(
"%.1024r has type bytes, but isn't valid UTF-8 "
'encoding. Non-UTF-8 strings must be converted to '
'unicode objects before being added.' % (proposed_value)
)
else:
try:
proposed_value.encode('utf8')
except UnicodeEncodeError:
raise ValueError('%.1024r isn\'t a valid unicode string and '
'can\'t be encoded in UTF-8.'%
(proposed_value))
raise ValueError(
"%.1024r isn't a valid unicode string and "
"can't be encoded in UTF-8." % (proposed_value)
)
return proposed_value
def DefaultValue(self):
return u""
return ''
class Int32ValueChecker(IntValueChecker):
@ -273,12 +297,17 @@ class DoubleValueChecker(object):
def CheckValue(self, proposed_value):
"""Check and convert proposed_value to float."""
if (not hasattr(proposed_value, '__float__') and
not hasattr(proposed_value, '__index__')) or (
type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'ndarray'):
message = ('%.1024r has type %s, but expected one of: int, float' %
(proposed_value, type(proposed_value)))
if (
not hasattr(proposed_value, '__float__')
and not hasattr(proposed_value, '__index__')
) or (
type(proposed_value).__module__ == 'numpy'
and type(proposed_value).__name__ == 'ndarray'
):
message = '%.1024r has type %s, but expected one of: int, float' % (
proposed_value,
type(proposed_value),
)
raise TypeError(message)
return float(proposed_value)
@ -309,6 +338,7 @@ class FloatValueChecker(DoubleValueChecker):
return TruncateToFourByteFloat(converted_value)
# Type-checkers for all scalar CPPTYPEs.
_VALUE_CHECKERS = {
_FieldDescriptor.CPPTYPE_INT32: Int32ValueChecker(),
@ -321,7 +351,6 @@ _VALUE_CHECKERS = {
_FieldDescriptor.CPPTYPE_STRING: TypeCheckerWithDefault(b'', bytes),
}
# Map from field type to a function F, such that F(field_num, value)
# gives the total byte size for a value of the given type. This
# byte size includes tag information and any other additional space
@ -344,9 +373,8 @@ TYPE_TO_BYTE_SIZE_FN = {
_FieldDescriptor.TYPE_SFIXED32: wire_format.SFixed32ByteSize,
_FieldDescriptor.TYPE_SFIXED64: wire_format.SFixed64ByteSize,
_FieldDescriptor.TYPE_SINT32: wire_format.SInt32ByteSize,
_FieldDescriptor.TYPE_SINT64: wire_format.SInt64ByteSize
}
_FieldDescriptor.TYPE_SINT64: wire_format.SInt64ByteSize,
}
# Maps from field types to encoder constructors.
TYPE_TO_ENCODER = {
@ -368,8 +396,7 @@ TYPE_TO_ENCODER = {
_FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Encoder,
_FieldDescriptor.TYPE_SINT32: encoder.SInt32Encoder,
_FieldDescriptor.TYPE_SINT64: encoder.SInt64Encoder,
}
}
# Maps from field types to sizer constructors.
TYPE_TO_SIZER = {
@ -391,8 +418,7 @@ TYPE_TO_SIZER = {
_FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Sizer,
_FieldDescriptor.TYPE_SINT32: encoder.SInt32Sizer,
_FieldDescriptor.TYPE_SINT64: encoder.SInt64Sizer,
}
}
# Maps from field type to a decoder constructor.
TYPE_TO_DECODER = {
@ -414,7 +440,7 @@ TYPE_TO_DECODER = {
_FieldDescriptor.TYPE_SFIXED64: decoder.SFixed64Decoder,
_FieldDescriptor.TYPE_SINT32: decoder.SInt32Decoder,
_FieldDescriptor.TYPE_SINT64: decoder.SInt64Decoder,
}
}
# Maps from field type to expected wiretype.
FIELD_TYPE_TO_WIRE_TYPE = {
@ -426,17 +452,14 @@ FIELD_TYPE_TO_WIRE_TYPE = {
_FieldDescriptor.TYPE_FIXED64: wire_format.WIRETYPE_FIXED64,
_FieldDescriptor.TYPE_FIXED32: wire_format.WIRETYPE_FIXED32,
_FieldDescriptor.TYPE_BOOL: wire_format.WIRETYPE_VARINT,
_FieldDescriptor.TYPE_STRING:
wire_format.WIRETYPE_LENGTH_DELIMITED,
_FieldDescriptor.TYPE_STRING: wire_format.WIRETYPE_LENGTH_DELIMITED,
_FieldDescriptor.TYPE_GROUP: wire_format.WIRETYPE_START_GROUP,
_FieldDescriptor.TYPE_MESSAGE:
wire_format.WIRETYPE_LENGTH_DELIMITED,
_FieldDescriptor.TYPE_BYTES:
wire_format.WIRETYPE_LENGTH_DELIMITED,
_FieldDescriptor.TYPE_MESSAGE: wire_format.WIRETYPE_LENGTH_DELIMITED,
_FieldDescriptor.TYPE_BYTES: wire_format.WIRETYPE_LENGTH_DELIMITED,
_FieldDescriptor.TYPE_UINT32: wire_format.WIRETYPE_VARINT,
_FieldDescriptor.TYPE_ENUM: wire_format.WIRETYPE_VARINT,
_FieldDescriptor.TYPE_SFIXED32: wire_format.WIRETYPE_FIXED32,
_FieldDescriptor.TYPE_SFIXED64: wire_format.WIRETYPE_FIXED64,
_FieldDescriptor.TYPE_SINT32: wire_format.WIRETYPE_VARINT,
_FieldDescriptor.TYPE_SINT64: wire_format.WIRETYPE_VARINT,
}
}

121
python/google/protobuf/internal/unknown_fields_test.py
View file

@ -29,6 +29,7 @@ from google.protobuf import map_unittest_pb2
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
try:
import tracemalloc # pylint: disable=g-import-not-at-top
except ImportError:
@ -90,8 +91,9 @@ class UnknownFieldsTest(unittest.TestCase):
# Unknown field should have wire format data which can be parsed back to
# original message.
self.assertEqual(unknown_field_set[0].field_number, item.type_id)
self.assertEqual(unknown_field_set[0].wire_type,
wire_format.WIRETYPE_LENGTH_DELIMITED)
self.assertEqual(
unknown_field_set[0].wire_type, wire_format.WIRETYPE_LENGTH_DELIMITED
)
d = unknown_field_set[0].data
message_new = message_set_extensions_pb2.TestMessageSetExtension1()
message_new.ParseFromString(d)
@ -120,29 +122,37 @@ class UnknownFieldsTest(unittest.TestCase):
other_message = unittest_pb2.TestAllTypes()
other_message.optional_string = 'discard'
message.optional_nested_message.ParseFromString(
other_message.SerializeToString())
other_message.SerializeToString()
)
message.repeated_nested_message.add().ParseFromString(
other_message.SerializeToString())
other_message.SerializeToString()
)
self.assertNotEqual(
b'', message.optional_nested_message.SerializeToString())
b'', message.optional_nested_message.SerializeToString()
)
self.assertNotEqual(
b'', message.repeated_nested_message[0].SerializeToString())
b'', message.repeated_nested_message[0].SerializeToString()
)
message.DiscardUnknownFields()
self.assertEqual(b'', message.optional_nested_message.SerializeToString())
self.assertEqual(
b'', message.repeated_nested_message[0].SerializeToString())
b'', message.repeated_nested_message[0].SerializeToString()
)
msg = map_unittest_pb2.TestMap()
msg.map_int32_all_types[1].optional_nested_message.ParseFromString(
other_message.SerializeToString())
other_message.SerializeToString()
)
msg.map_string_string['1'] = 'test'
self.assertNotEqual(
b'',
msg.map_int32_all_types[1].optional_nested_message.SerializeToString())
msg.map_int32_all_types[1].optional_nested_message.SerializeToString(),
)
msg.DiscardUnknownFields()
self.assertEqual(
b'',
msg.map_int32_all_types[1].optional_nested_message.SerializeToString())
msg.map_int32_all_types[1].optional_nested_message.SerializeToString(),
)
def testUnknownFieldsInExtension(self):
msg = message_set_extensions_pb2.TestMessageSet()
@ -169,15 +179,15 @@ class UnknownFieldsAccessorsTest(unittest.TestCase):
def CheckUnknownField(self, name, unknown_field_set, expected_value):
field_descriptor = self.descriptor.fields_by_name[name]
expected_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[
field_descriptor.type]
expected_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type]
for unknown_field in unknown_field_set:
if unknown_field.field_number == field_descriptor.number:
self.assertEqual(expected_type, unknown_field.wire_type)
if expected_type == 3:
# Check group
self.assertEqual(expected_value[0],
unknown_field.data[0].field_number)
self.assertEqual(
expected_value[0], unknown_field.data[0].field_number
)
self.assertEqual(expected_value[1], unknown_field.data[0].wire_type)
self.assertEqual(expected_value[2], unknown_field.data[0].data)
continue
@ -191,39 +201,43 @@ class UnknownFieldsAccessorsTest(unittest.TestCase):
def testCheckUnknownFieldValue(self):
unknown_field_set = unknown_fields.UnknownFieldSet(self.empty_message)
# Test enum.
self.CheckUnknownField('optional_nested_enum',
unknown_field_set,
self.all_fields.optional_nested_enum)
self.CheckUnknownField(
'optional_nested_enum',
unknown_field_set,
self.all_fields.optional_nested_enum,
)
# Test repeated enum.
self.CheckUnknownField('repeated_nested_enum',
unknown_field_set,
self.all_fields.repeated_nested_enum)
self.CheckUnknownField(
'repeated_nested_enum',
unknown_field_set,
self.all_fields.repeated_nested_enum,
)
# Test varint.
self.CheckUnknownField('optional_int32',
unknown_field_set,
self.all_fields.optional_int32)
self.CheckUnknownField(
'optional_int32', unknown_field_set, self.all_fields.optional_int32
)
# Test fixed32.
self.CheckUnknownField('optional_fixed32',
unknown_field_set,
self.all_fields.optional_fixed32)
self.CheckUnknownField(
'optional_fixed32', unknown_field_set, self.all_fields.optional_fixed32
)
# Test fixed64.
self.CheckUnknownField('optional_fixed64',
unknown_field_set,
self.all_fields.optional_fixed64)
self.CheckUnknownField(
'optional_fixed64', unknown_field_set, self.all_fields.optional_fixed64
)
# Test length delimited.
self.CheckUnknownField('optional_string',
unknown_field_set,
self.all_fields.optional_string.encode('utf-8'))
self.CheckUnknownField(
'optional_string',
unknown_field_set,
self.all_fields.optional_string.encode('utf-8'),
)
# Test group.
self.CheckUnknownField('optionalgroup',
unknown_field_set,
(17, 0, 117))
self.CheckUnknownField('optionalgroup', unknown_field_set, (17, 0, 117))
self.assertEqual(99, len(unknown_field_set))
@ -265,8 +279,10 @@ class UnknownFieldsAccessorsTest(unittest.TestCase):
self.assertEqual(self.empty_message.SerializeToString(), b'')
self.assertEqual(len(unknown_field_set), 99)
@unittest.skipIf((sys.version_info.major, sys.version_info.minor) < (3, 4),
'tracemalloc requires python 3.4+')
@unittest.skipIf(
(sys.version_info.major, sys.version_info.minor) < (3, 4),
'tracemalloc requires python 3.4+',
)
def testUnknownFieldsNoMemoryLeak(self):
# Call to UnknownFields must not leak memory
nb_leaks = 1234
@ -300,14 +316,17 @@ class UnknownFieldsAccessorsTest(unittest.TestCase):
message.optional_uint32 = 456
nested_message = unittest_pb2.NestedTestAllTypes()
nested_message.payload.optional_nested_message.ParseFromString(
message.SerializeToString())
message.SerializeToString()
)
unknown_field_set = unknown_fields.UnknownFieldSet(
nested_message.payload.optional_nested_message)
nested_message.payload.optional_nested_message
)
self.assertEqual(unknown_field_set[0].data, 456)
nested_message.ClearField('payload')
self.assertEqual(unknown_field_set[0].data, 456)
unknown_field_set = unknown_fields.UnknownFieldSet(
nested_message.payload.optional_nested_message)
nested_message.payload.optional_nested_message
)
self.assertEqual(0, len(unknown_field_set))
def testUnknownField(self):
@ -335,15 +354,16 @@ class UnknownEnumValuesTest(unittest.TestCase):
self.message = missing_enum_values_pb2.TestEnumValues()
# TestEnumValues.ZERO = 0, but does not exist in the other NestedEnum.
self.message.optional_nested_enum = (
missing_enum_values_pb2.TestEnumValues.ZERO)
missing_enum_values_pb2.TestEnumValues.ZERO
)
self.message.repeated_nested_enum.extend([
missing_enum_values_pb2.TestEnumValues.ZERO,
missing_enum_values_pb2.TestEnumValues.ONE,
])
])
self.message.packed_nested_enum.extend([
missing_enum_values_pb2.TestEnumValues.ZERO,
missing_enum_values_pb2.TestEnumValues.ONE,
])
])
self.message_data = self.message.SerializeToString()
self.missing_message = missing_enum_values_pb2.TestMissingEnumValues()
self.missing_message.ParseFromString(self.message_data)
@ -401,12 +421,15 @@ class UnknownEnumValuesTest(unittest.TestCase):
def testCheckUnknownFieldValueForEnum(self):
unknown_field_set = unknown_fields.UnknownFieldSet(self.missing_message)
self.assertEqual(len(unknown_field_set), 5)
self.CheckUnknownField('optional_nested_enum',
self.message.optional_nested_enum)
self.CheckUnknownField('repeated_nested_enum',
self.message.repeated_nested_enum)
self.CheckUnknownField('packed_nested_enum',
self.message.packed_nested_enum)
self.CheckUnknownField(
'optional_nested_enum', self.message.optional_nested_enum
)
self.CheckUnknownField(
'repeated_nested_enum', self.message.repeated_nested_enum
)
self.CheckUnknownField(
'packed_nested_enum', self.message.packed_nested_enum
)
def testRoundTrip(self):
new_message = missing_enum_values_pb2.TestEnumValues()

2
python/google/protobuf/internal/well_known_types.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains well known classes.
This files defines well known classes which need extra maintenance including:
@ -696,7 +695,6 @@ class ListValue(object):
collections.abc.MutableSequence.register(ListValue)
# LINT.IfChange(wktbases)
WKTBASES = {
'google.protobuf.Any': Any,

72
python/google/protobuf/internal/wire_format.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Constants and static functions to support protocol buffer wire format."""
__author__ = 'robinson@google.com (Will Robinson)'
@ -13,7 +12,6 @@ import struct
from google.protobuf import descriptor
from google.protobuf import message
TAG_TYPE_BITS = 3 # Number of bits used to hold type info in a proto tag.
TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1 # 0x7
@ -29,7 +27,6 @@ WIRETYPE_END_GROUP = 4
WIRETYPE_FIXED32 = 5
_WIRETYPE_MAX = 5
# Bounds for various integer types.
INT32_MAX = int((1 << 31) - 1)
INT32_MIN = int(-(1 << 31))
@ -45,7 +42,6 @@ FORMAT_UINT64_LITTLE_ENDIAN = '<Q'
FORMAT_FLOAT_LITTLE_ENDIAN = '<f'
FORMAT_DOUBLE_LITTLE_ENDIAN = '<d'
# We'll have to provide alternate implementations of AppendLittleEndian*() on
# any architectures where these checks fail.
if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4:
@ -56,6 +52,7 @@ if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8:
def PackTag(field_number, wire_type):
"""Returns an unsigned 32-bit integer that encodes the field number and
wire type information in standard protocol message wire format.
Args:
@ -68,14 +65,16 @@ def PackTag(field_number, wire_type):
def UnpackTag(tag):
"""The inverse of PackTag(). Given an unsigned 32-bit number,
returns a (field_number, wire_type) tuple.
"""The inverse of PackTag().
Given an unsigned 32-bit number, returns a (field_number, wire_type) tuple.
"""
return (tag >> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK)
def ZigZagEncode(value):
"""ZigZag Transform: Encodes signed integers so that they can be
effectively used with varint encoding. See wire_format.h for
more details.
"""
@ -91,7 +90,6 @@ def ZigZagDecode(value):
return (value >> 1) ^ (~0)
# The *ByteSize() functions below return the number of bytes required to
# serialize "field number + type" information and then serialize the value.
@ -101,12 +99,12 @@ def Int32ByteSize(field_number, int32):
def Int32ByteSizeNoTag(int32):
return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32)
return _VarUInt64ByteSizeNoTag(0xFFFFFFFFFFFFFFFF & int32)
def Int64ByteSize(field_number, int64):
# Have to convert to uint before calling UInt64ByteSize().
return UInt64ByteSize(field_number, 0xffffffffffffffff & int64)
return UInt64ByteSize(field_number, 0xFFFFFFFFFFFFFFFF & int64)
def UInt32ByteSize(field_number, uint32):
@ -162,20 +160,21 @@ def StringByteSize(field_number, string):
def BytesByteSize(field_number, b):
return (TagByteSize(field_number)
+ _VarUInt64ByteSizeNoTag(len(b))
+ len(b))
return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(len(b)) + len(b)
def GroupByteSize(field_number, message):
return (2 * TagByteSize(field_number) # START and END group.
+ message.ByteSize())
return (
2 * TagByteSize(field_number) + message.ByteSize() # START and END group.
)
def MessageByteSize(field_number, message):
return (TagByteSize(field_number)
+ _VarUInt64ByteSizeNoTag(message.ByteSize())
+ message.ByteSize())
return (
TagByteSize(field_number)
+ _VarUInt64ByteSizeNoTag(message.ByteSize())
+ message.ByteSize()
)
def MessageSetItemByteSize(field_number, msg):
@ -183,7 +182,7 @@ def MessageSetItemByteSize(field_number, msg):
# There are 2 tags for the beginning and ending of the repeated group, that
# is field number 1, one with field number 2 (type_id) and one with field
# number 3 (message).
total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3))
total_size = 2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3)
# Add the number of bytes for type_id.
total_size += _VarUInt64ByteSizeNoTag(field_number)
@ -206,30 +205,41 @@ def TagByteSize(field_number):
# Private helper function for the *ByteSize() functions above.
def _VarUInt64ByteSizeNoTag(uint64):
"""Returns the number of bytes required to serialize a single varint
using boundary value comparisons. (unrolled loop optimization -WPierce)
uint64 must be unsigned.
"""
if uint64 <= 0x7f: return 1
if uint64 <= 0x3fff: return 2
if uint64 <= 0x1fffff: return 3
if uint64 <= 0xfffffff: return 4
if uint64 <= 0x7ffffffff: return 5
if uint64 <= 0x3ffffffffff: return 6
if uint64 <= 0x1ffffffffffff: return 7
if uint64 <= 0xffffffffffffff: return 8
if uint64 <= 0x7fffffffffffffff: return 9
if uint64 <= 0x7F:
return 1
if uint64 <= 0x3FFF:
return 2
if uint64 <= 0x1FFFFF:
return 3
if uint64 <= 0xFFFFFFF:
return 4
if uint64 <= 0x7FFFFFFFF:
return 5
if uint64 <= 0x3FFFFFFFFFF:
return 6
if uint64 <= 0x1FFFFFFFFFFFF:
return 7
if uint64 <= 0xFFFFFFFFFFFFFF:
return 8
if uint64 <= 0x7FFFFFFFFFFFFFFF:
return 9
if uint64 > UINT64_MAX:
raise message.EncodeError('Value out of range: %d' % uint64)
return 10
NON_PACKABLE_TYPES = (
descriptor.FieldDescriptor.TYPE_STRING,
descriptor.FieldDescriptor.TYPE_GROUP,
descriptor.FieldDescriptor.TYPE_MESSAGE,
descriptor.FieldDescriptor.TYPE_BYTES
descriptor.FieldDescriptor.TYPE_STRING,
descriptor.FieldDescriptor.TYPE_GROUP,
descriptor.FieldDescriptor.TYPE_MESSAGE,
descriptor.FieldDescriptor.TYPE_BYTES,
)

101
python/google/protobuf/internal/wire_format_test.py
View file

@ -18,10 +18,12 @@ from google.protobuf.internal import wire_format
class WireFormatTest(unittest.TestCase):
def testPackTag(self):
field_number = 0xabc
field_number = 0xABC
tag_type = 2
self.assertEqual((field_number << 3) | tag_type,
wire_format.PackTag(field_number, tag_type))
self.assertEqual(
(field_number << 3) | tag_type,
wire_format.PackTag(field_number, tag_type),
)
PackTag = wire_format.PackTag
# Number too high.
self.assertRaises(message.EncodeError, PackTag, field_number, 6)
@ -33,7 +35,8 @@ class WireFormatTest(unittest.TestCase):
for expected_field_number in (1, 15, 16, 2047, 2048):
for expected_wire_type in range(6): # Highest-numbered wiretype is 5.
field_number, wire_type = wire_format.UnpackTag(
wire_format.PackTag(expected_field_number, expected_wire_type))
wire_format.PackTag(expected_field_number, expected_wire_type)
)
self.assertEqual(expected_field_number, field_number)
self.assertEqual(expected_wire_type, wire_type)
@ -49,10 +52,10 @@ class WireFormatTest(unittest.TestCase):
self.assertEqual(2, Z(1))
self.assertEqual(3, Z(-2))
self.assertEqual(4, Z(2))
self.assertEqual(0xfffffffe, Z(0x7fffffff))
self.assertEqual(0xffffffff, Z(-0x80000000))
self.assertEqual(0xfffffffffffffffe, Z(0x7fffffffffffffff))
self.assertEqual(0xffffffffffffffff, Z(-0x8000000000000000))
self.assertEqual(0xFFFFFFFE, Z(0x7FFFFFFF))
self.assertEqual(0xFFFFFFFF, Z(-0x80000000))
self.assertEqual(0xFFFFFFFFFFFFFFFE, Z(0x7FFFFFFFFFFFFFFF))
self.assertEqual(0xFFFFFFFFFFFFFFFF, Z(-0x8000000000000000))
self.assertRaises(TypeError, Z, None)
self.assertRaises(TypeError, Z, 'abcd')
@ -66,10 +69,10 @@ class WireFormatTest(unittest.TestCase):
self.assertEqual(1, Z(2))
self.assertEqual(-2, Z(3))
self.assertEqual(2, Z(4))
self.assertEqual(0x7fffffff, Z(0xfffffffe))
self.assertEqual(-0x80000000, Z(0xffffffff))
self.assertEqual(0x7fffffffffffffff, Z(0xfffffffffffffffe))
self.assertEqual(-0x8000000000000000, Z(0xffffffffffffffff))
self.assertEqual(0x7FFFFFFF, Z(0xFFFFFFFE))
self.assertEqual(-0x80000000, Z(0xFFFFFFFF))
self.assertEqual(0x7FFFFFFFFFFFFFFF, Z(0xFFFFFFFFFFFFFFFE))
self.assertEqual(-0x8000000000000000, Z(0xFFFFFFFFFFFFFFFF))
self.assertRaises(TypeError, Z, None)
self.assertRaises(TypeError, Z, 'abcd')
@ -81,10 +84,13 @@ class WireFormatTest(unittest.TestCase):
for field_number, tag_bytes in ((15, 1), (16, 2), (2047, 2), (2048, 3)):
expected_size = expected_value_size + tag_bytes
actual_size = byte_size_fn(field_number, value)
self.assertEqual(expected_size, actual_size,
'byte_size_fn: %s, field_number: %d, value: %r\n'
'Expected: %d, Actual: %d'% (
byte_size_fn, field_number, value, expected_size, actual_size))
self.assertEqual(
expected_size,
actual_size,
'byte_size_fn: %s, field_number: %d, value: %r\n'
'Expected: %d, Actual: %d'
% (byte_size_fn, field_number, value, expected_size, actual_size),
)
def testByteSizeFunctions(self):
# Test all numeric *ByteSize() functions.
@ -155,7 +161,7 @@ class WireFormatTest(unittest.TestCase):
[wire_format.EnumByteSize, 127, 1],
[wire_format.EnumByteSize, 128, 2],
[wire_format.EnumByteSize, wire_format.UINT32_MAX, 5],
]
]
for args in NUMERIC_ARGS:
self.NumericByteSizeTestHelper(*args)
@ -170,12 +176,18 @@ class WireFormatTest(unittest.TestCase):
# Test UTF-8 string byte size calculation.
# 1 byte for tag, 1 byte for length, 8 bytes for content.
self.assertEqual(10, wire_format.StringByteSize(
5, b'\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82'.decode('utf-8')))
self.assertEqual(
10,
wire_format.StringByteSize(
5, b'\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82'.decode('utf-8')
),
)
class MockMessage(object):
def __init__(self, byte_size):
self.byte_size = byte_size
def ByteSize(self):
return self.byte_size
@ -183,46 +195,59 @@ class WireFormatTest(unittest.TestCase):
mock_message = MockMessage(byte_size=message_byte_size)
# Test groups.
# (2 * 1) bytes for begin and end tags, plus message_byte_size.
self.assertEqual(2 + message_byte_size,
wire_format.GroupByteSize(1, mock_message))
self.assertEqual(
2 + message_byte_size, wire_format.GroupByteSize(1, mock_message)
)
# (2 * 2) bytes for begin and end tags, plus message_byte_size.
self.assertEqual(4 + message_byte_size,
wire_format.GroupByteSize(16, mock_message))
self.assertEqual(
4 + message_byte_size, wire_format.GroupByteSize(16, mock_message)
)
# Test messages.
# 1 byte for tag, plus 1 byte for length, plus contents.
self.assertEqual(2 + mock_message.byte_size,
wire_format.MessageByteSize(1, mock_message))
self.assertEqual(
2 + mock_message.byte_size, wire_format.MessageByteSize(1, mock_message)
)
# 2 bytes for tag, plus 1 byte for length, plus contents.
self.assertEqual(3 + mock_message.byte_size,
wire_format.MessageByteSize(16, mock_message))
self.assertEqual(
3 + mock_message.byte_size,
wire_format.MessageByteSize(16, mock_message),
)
# 2 bytes for tag, plus 2 bytes for length, plus contents.
mock_message.byte_size = 128
self.assertEqual(4 + mock_message.byte_size,
wire_format.MessageByteSize(16, mock_message))
self.assertEqual(
4 + mock_message.byte_size,
wire_format.MessageByteSize(16, mock_message),
)
# Test message set item byte size.
# 4 bytes for tags, plus 1 byte for length, plus 1 byte for type_id,
# plus contents.
mock_message.byte_size = 10
self.assertEqual(mock_message.byte_size + 6,
wire_format.MessageSetItemByteSize(1, mock_message))
self.assertEqual(
mock_message.byte_size + 6,
wire_format.MessageSetItemByteSize(1, mock_message),
)
# 4 bytes for tags, plus 2 bytes for length, plus 1 byte for type_id,
# plus contents.
mock_message.byte_size = 128
self.assertEqual(mock_message.byte_size + 7,
wire_format.MessageSetItemByteSize(1, mock_message))
self.assertEqual(
mock_message.byte_size + 7,
wire_format.MessageSetItemByteSize(1, mock_message),
)
# 4 bytes for tags, plus 2 bytes for length, plus 2 byte for type_id,
# plus contents.
self.assertEqual(mock_message.byte_size + 8,
wire_format.MessageSetItemByteSize(128, mock_message))
self.assertEqual(
mock_message.byte_size + 8,
wire_format.MessageSetItemByteSize(128, mock_message),
)
# Too-long varint.
self.assertRaises(message.EncodeError,
wire_format.UInt64ByteSize, 1, 1 << 128)
self.assertRaises(
message.EncodeError, wire_format.UInt64ByteSize, 1, 1 << 128
)
if __name__ == '__main__':

19
python/google/protobuf/json_format.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains routines for printing protocol messages in JSON format.
Simple usage example:
@ -19,7 +18,6 @@ Simple usage example:
__author__ = 'jieluo@google.com (Jie Luo)'
import base64
from collections import OrderedDict
import json
@ -32,7 +30,6 @@ from google.protobuf import message_factory
from google.protobuf import symbol_database
from google.protobuf.internal import type_checkers
_INT_TYPES = frozenset([
descriptor.FieldDescriptor.CPPTYPE_INT32,
descriptor.FieldDescriptor.CPPTYPE_UINT32,
@ -109,9 +106,9 @@ def MessageToJson(
default.
ensure_ascii: If True, strings with non-ASCII characters are escaped. If
False, Unicode strings are returned unchanged.
unquote_int64_if_possible: If True, unquote int64 fields for values that
are safe to emit as numbers (all values smaller than 2^53 and a sparse
set of values that are larger).
unquote_int64_if_possible: If True, unquote int64 fields for values that are
safe to emit as numbers (all values smaller than 2^53 and a sparse set of
values that are larger).
Returns:
A string containing the JSON formatted protocol buffer message.
@ -151,9 +148,9 @@ def MessageToDict(
use_integers_for_enums: If true, print integers instead of enum names.
descriptor_pool: A Descriptor Pool for resolving types. If None use the
default.
unquote_int64_if_possible: If True, unquote int64 fields for values that
are safe to emit as numbers (all values smaller than 2^53 and a sparse
set of values that are larger).
unquote_int64_if_possible: If True, unquote int64 fields for values that are
safe to emit as numbers (all values smaller than 2^53 and a sparse set of
values that are larger).
Returns:
A dict representation of the protocol buffer message.
@ -750,9 +747,7 @@ class _Parser(object):
elif full_name in _WKTJSONMETHODS:
# For well-known types (including nested Any), use ConvertMessage
# to ensure recursion depth is properly tracked
self.ConvertMessage(
value['value'], sub_message, '{0}.value'.format(path)
)
self.ConvertMessage(value['value'], sub_message, '{0}.value'.format(path))
else:
del value['@type']
try:

18
python/google/protobuf/message.py
View file

@ -7,8 +7,6 @@
# TODO: We should just make these methods all "pure-virtual" and move
# all implementation out, into reflection.py for now.
"""Contains an abstract base class for protocol messages."""
__author__ = 'robinson@google.com (Will Robinson)'
@ -18,21 +16,23 @@ _INCONSISTENT_MESSAGE_ATTRIBUTES = ('Extensions',)
class Error(Exception):
"""Base error type for this module."""
pass
class DecodeError(Error):
"""Exception raised when deserializing messages."""
pass
class EncodeError(Error):
"""Exception raised when serializing messages."""
pass
class Message(object):
"""Abstract base class for protocol messages.
Protocol message classes are almost always generated by the protocol
@ -198,8 +198,8 @@ class Message(object):
Args:
serialized (bytes): Any object that allows us to call
``memoryview(serialized)`` to access a string of bytes using the
buffer interface.
``memoryview(serialized)`` to access a string of bytes using the buffer
interface.
Returns:
int: The number of bytes read from `serialized`.
@ -396,6 +396,7 @@ class Message(object):
def _SetListener(self, message_listener):
"""Internal method used by the protocol message implementation.
Clients should not call this directly.
Sets a listener that this message will call on certain state transitions.
@ -437,8 +438,11 @@ class Message(object):
# Python does not pickle nested classes; use the symbol_database on the
# receiving end.
container = message_descriptor
return (_InternalConstructMessage, (container.full_name,),
self.__getstate__())
return (
_InternalConstructMessage,
(container.full_name,),
self.__getstate__(),
)
def _InternalConstructMessage(full_name):

2
python/google/protobuf/message_factory.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Provides a factory class for generating dynamic messages.
The easiest way to use this class is if you have access to the FileDescriptor
@ -27,7 +26,6 @@ if api_implementation.Type() == 'python':
else:
from google.protobuf.pyext import cpp_message as message_impl # pylint: disable=g-import-not-at-top
# The type of all Message classes.
_GENERATED_PROTOCOL_MESSAGE_TYPE = message_impl.GeneratedProtocolMessageType

5
python/google/protobuf/proto.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains the Nextgen Pythonic protobuf APIs."""
import io
@ -22,8 +21,8 @@ def serialize(message: _MESSAGE, deterministic: bool = None) -> bytes:
Args:
message: The proto message to be serialized.
deterministic: If true, requests deterministic serialization
of the protobuf, with predictable ordering of map keys.
deterministic: If true, requests deterministic serialization of the
protobuf, with predictable ordering of map keys.
Returns:
A binary bytes representation of the message.

24
python/google/protobuf/proto_builder.py
View file

@ -4,15 +4,14 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Dynamic Protobuf class creator."""
from collections import OrderedDict
import hashlib
import os
from google.protobuf import descriptor_pb2
from google.protobuf import descriptor
from google.protobuf import descriptor_pb2
from google.protobuf import descriptor_pool
from google.protobuf import message_factory
@ -23,6 +22,7 @@ def _GetMessageFromFactory(pool, full_name):
Args:
pool: a descriptor pool.
full_name: str, the fully qualified name of the proto type.
Returns:
A class, for the type identified by full_name.
Raises:
@ -40,10 +40,11 @@ def MakeSimpleProtoClass(fields, full_name=None, pool=None):
Args:
fields: dict of {name: field_type} mappings for each field in the proto. If
this is an OrderedDict the order will be maintained, otherwise the
fields will be sorted by name.
this is an OrderedDict the order will be maintained, otherwise the fields
will be sorted by name.
full_name: optional str, the fully-qualified name of the proto type.
pool: optional DescriptorPool instance.
Returns:
a class, the new protobuf class with a FileDescriptor.
"""
@ -73,8 +74,10 @@ def MakeSimpleProtoClass(fields, full_name=None, pool=None):
# If the proto is anonymous, use the same hash to name it.
if full_name is None:
full_name = ('net.proto2.python.public.proto_builder.AnonymousProto_' +
fields_hash.hexdigest())
full_name = (
'net.proto2.python.public.proto_builder.AnonymousProto_'
+ fields_hash.hexdigest()
)
try:
proto_cls = _GetMessageFromFactory(pool_instance, full_name)
return proto_cls
@ -84,7 +87,8 @@ def MakeSimpleProtoClass(fields, full_name=None, pool=None):
# This is the first time we see this proto: add a new descriptor to the pool.
pool_instance.Add(
_MakeFileDescriptorProto(proto_file_name, full_name, field_items))
_MakeFileDescriptorProto(proto_file_name, full_name, field_items)
)
return _GetMessageFromFactory(pool_instance, full_name)
@ -103,8 +107,10 @@ def _MakeFileDescriptorProto(proto_file_name, full_name, field_items):
# # number after the range.
if f_number >= descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER:
f_number += (
descriptor.FieldDescriptor.LAST_RESERVED_FIELD_NUMBER -
descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER + 1)
descriptor.FieldDescriptor.LAST_RESERVED_FIELD_NUMBER
- descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER
+ 1
)
field_proto.number = f_number
field_proto.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL
field_proto.type = f_type

21
python/google/protobuf/proto_json.py
View file

@ -4,21 +4,21 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains the Nextgen Pythonic Protobuf JSON APIs."""
from typing import Optional, Type
from google.protobuf.message import Message
from google.protobuf.descriptor_pool import DescriptorPool
from google.protobuf import json_format
from google.protobuf.descriptor_pool import DescriptorPool
from google.protobuf.message import Message
def serialize(
message: Message,
always_print_fields_with_no_presence: bool=False,
preserving_proto_field_name: bool=False,
use_integers_for_enums: bool=False,
descriptor_pool: Optional[DescriptorPool]=None,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
descriptor_pool: Optional[DescriptorPool] = None,
) -> dict:
"""Converts protobuf message to a dictionary.
@ -47,12 +47,13 @@ def serialize(
use_integers_for_enums=use_integers_for_enums,
)
def parse(
message_class: Type[Message],
js_dict: dict,
ignore_unknown_fields: bool=False,
descriptor_pool: Optional[DescriptorPool]=None,
max_recursion_depth: int=100
ignore_unknown_fields: bool = False,
descriptor_pool: Optional[DescriptorPool] = None,
max_recursion_depth: int = 100,
) -> Message:
"""Parses a JSON dictionary representation into a message.

2
python/google/protobuf/proto_text.py
View file

@ -4,8 +4,8 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains the Nextgen Pythonic Protobuf Text Format APIs."""
from typing import AnyStr, Callable, Optional, Text, Type, Union
from google.protobuf import text_format

3
python/google/protobuf/pyext/cpp_message.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Protocol message implementation hooks for C++ implementation.
Contains helper functions used to create protocol message classes from
@ -15,7 +14,6 @@ __author__ = 'tibell@google.com (Johan Tibell)'
from google.protobuf.internal import api_implementation
# pylint: disable=protected-access
_message = api_implementation._c_module
# TODO: Remove this import after fix api_implementation
@ -24,7 +22,6 @@ if _message is None:
class GeneratedProtocolMessageType(_message.MessageMeta):
"""Metaclass for protocol message classes created at runtime from Descriptors.
The protocol compiler currently uses this metaclass to create protocol

2
python/google/protobuf/reflection.py
View file

@ -6,8 +6,8 @@
# https://developers.google.com/open-source/licenses/bsd
# This code is meant to work on Python 2.4 and above only.
"""Contains a metaclass and helper functions used to create
protocol message classes from Descriptor objects at runtime.
Recall that a metaclass is the "type" of a class.

3
python/google/protobuf/runtime_version.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Protobuf Runtime versions and validators.
It should only be accessed by Protobuf gencodes and tests. DO NOT USE it
@ -14,6 +13,7 @@ elsewhere.
__author__ = 'shaod@google.com (Dennis Shao)'
from enum import Enum
import os
import warnings
@ -42,6 +42,7 @@ SUFFIX = OSS_SUFFIX
_MAX_WARNING_COUNT = 20
_warning_count = 0
class VersionError(Exception):
"""Exception class for version violation."""

60
python/google/protobuf/service_reflection.py
View file

@ -6,6 +6,7 @@
# https://developers.google.com/open-source/licenses/bsd
"""Contains metaclasses used to create protocol service and service stub
classes from ServiceDescriptor objects at runtime.
The GeneratedServiceType and GeneratedServiceStubType metaclasses are used to
@ -17,7 +18,6 @@ __author__ = 'petar@google.com (Petar Petrov)'
class GeneratedServiceType(type):
"""Metaclass for service classes created at runtime from ServiceDescriptors.
Implementations for all methods described in the Service class are added here
@ -42,8 +42,7 @@ class GeneratedServiceType(type):
"""Creates a message service class.
Args:
name: Name of the class (ignored, but required by the metaclass
protocol).
name: Name of the class (ignored, but required by the metaclass protocol).
bases: Base classes of the class being constructed.
dictionary: The class dictionary of the class being constructed.
dictionary[_DESCRIPTOR_KEY] must contain a ServiceDescriptor object
@ -61,7 +60,6 @@ class GeneratedServiceType(type):
class GeneratedServiceStubType(GeneratedServiceType):
"""Metaclass for service stubs created at runtime from ServiceDescriptors.
This class has similar responsibilities as GeneratedServiceType, except that
@ -92,7 +90,6 @@ class GeneratedServiceStubType(GeneratedServiceType):
class _ServiceBuilder(object):
"""This class constructs a protocol service class using a service descriptor.
Given a service descriptor, this class constructs a class that represents
@ -105,8 +102,8 @@ class _ServiceBuilder(object):
"""Initializes an instance of the service class builder.
Args:
service_descriptor: ServiceDescriptor to use when constructing the
service class.
service_descriptor: ServiceDescriptor to use when constructing the service
class.
"""
self.descriptor = service_descriptor
@ -123,8 +120,9 @@ class _ServiceBuilder(object):
# Making sure to use exact argument names from the abstract interface in
# service.py to match the type signature
def _WrapCallMethod(self, method_descriptor, rpc_controller, request, done):
return builder._CallMethod(self, method_descriptor, rpc_controller,
request, done)
return builder._CallMethod(
self, method_descriptor, rpc_controller, request, done
)
def _WrapGetRequestClass(self, method_descriptor):
return builder._GetRequestClass(method_descriptor)
@ -141,8 +139,9 @@ class _ServiceBuilder(object):
for method in builder.descriptor.methods:
setattr(cls, method.name, builder._GenerateNonImplementedMethod(method))
def _CallMethod(self, srvc, method_descriptor,
rpc_controller, request, callback):
def _CallMethod(
self, srvc, method_descriptor, rpc_controller, request, callback
):
"""Calls the method described by a given method descriptor.
Args:
@ -154,7 +153,8 @@ class _ServiceBuilder(object):
"""
if method_descriptor.containing_service != self.descriptor:
raise RuntimeError(
'CallMethod() given method descriptor for wrong service type.')
'CallMethod() given method descriptor for wrong service type.'
)
method = getattr(srvc, method_descriptor.name)
return method(rpc_controller, request, callback)
@ -171,7 +171,8 @@ class _ServiceBuilder(object):
"""
if method_descriptor.containing_service != self.descriptor:
raise RuntimeError(
'GetRequestClass() given method descriptor for wrong service type.')
'GetRequestClass() given method descriptor for wrong service type.'
)
return method_descriptor.input_type._concrete_class
def _GetResponseClass(self, method_descriptor):
@ -187,7 +188,8 @@ class _ServiceBuilder(object):
"""
if method_descriptor.containing_service != self.descriptor:
raise RuntimeError(
'GetResponseClass() given method descriptor for wrong service type.')
'GetResponseClass() given method descriptor for wrong service type.'
)
return method_descriptor.output_type._concrete_class
def _GenerateNonImplementedMethod(self, method):
@ -201,7 +203,8 @@ class _ServiceBuilder(object):
A method that can be added to the service class.
"""
return lambda inst, rpc_controller, request, callback: (
self._NonImplementedMethod(method.name, rpc_controller, callback))
self._NonImplementedMethod(method.name, rpc_controller, callback)
)
def _NonImplementedMethod(self, method_name, rpc_controller, callback):
"""The body of all methods in the generated service class.
@ -216,7 +219,6 @@ class _ServiceBuilder(object):
class _ServiceStubBuilder(object):
"""Constructs a protocol service stub class using a service descriptor.
Given a service descriptor, this class constructs a suitable stub class.
@ -231,8 +233,8 @@ class _ServiceStubBuilder(object):
"""Initializes an instance of the service stub class builder.
Args:
service_descriptor: ServiceDescriptor to use when constructing the
stub class.
service_descriptor: ServiceDescriptor to use when constructing the stub
class.
"""
self.descriptor = service_descriptor
@ -245,17 +247,22 @@ class _ServiceStubBuilder(object):
def _ServiceStubInit(stub, rpc_channel):
stub.rpc_channel = rpc_channel
self.cls = cls
cls.__init__ = _ServiceStubInit
for method in self.descriptor.methods:
setattr(cls, method.name, self._GenerateStubMethod(method))
def _GenerateStubMethod(self, method):
return (lambda inst, rpc_controller, request, callback=None:
self._StubMethod(inst, method, rpc_controller, request, callback))
return (
lambda inst, rpc_controller, request, callback=None: self._StubMethod(
inst, method, rpc_controller, request, callback
)
)
def _StubMethod(self, stub, method_descriptor,
rpc_controller, request, callback):
def _StubMethod(
self, stub, method_descriptor, rpc_controller, request, callback
):
"""The body of all service methods in the generated stub class.
Args:
@ -264,9 +271,14 @@ class _ServiceStubBuilder(object):
rpc_controller: Rpc controller to execute the method.
request: Request protocol message.
callback: A callback to execute when the method finishes.
Returns:
Response message (in case of blocking call).
"""
return stub.rpc_channel.CallMethod(
method_descriptor, rpc_controller, request,
method_descriptor.output_type._concrete_class, callback)
method_descriptor,
rpc_controller,
request,
method_descriptor.output_type._concrete_class,
callback,
)

5
python/google/protobuf/symbol_database.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""A database of Python protocol buffer generated symbols.
SymbolDatabase is the MessageFactory for messages generated at compile time,
@ -36,12 +35,12 @@ Example usage::
import warnings
from google.protobuf.internal import api_implementation
from google.protobuf import descriptor_pool
from google.protobuf import message_factory
from google.protobuf.internal import api_implementation
class SymbolDatabase():
class SymbolDatabase:
"""A database of Python generated symbols."""
# local cache of registered classes.

19
python/google/protobuf/text_encoding.py
View file

@ -4,13 +4,15 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Encoding related utilities."""
import re
def _AsciiIsPrint(i):
return i >= 32 and i < 127
def _MakeStrEscapes():
ret = {}
for i in range(0, 128):
@ -20,10 +22,11 @@ def _MakeStrEscapes():
ret[ord('\n')] = r'\n' # optional escape
ret[ord('\r')] = r'\r' # optional escape
ret[ord('"')] = r'\"' # necessary escape
ret[ord('\'')] = r"\'" # optional escape
ret[ord("'")] = r'\'' # optional escape
ret[ord('\\')] = r'\\' # necessary escape
return ret
# Maps int -> char, performing string escapes.
_str_escapes = _MakeStrEscapes()
@ -40,9 +43,9 @@ def _DecodeUtf8EscapeErrors(text_bytes):
ret += text_bytes.decode('utf-8').translate(_str_escapes)
text_bytes = ''
except UnicodeDecodeError as e:
ret += text_bytes[:e.start].decode('utf-8').translate(_str_escapes)
ret += text_bytes[: e.start].decode('utf-8').translate(_str_escapes)
ret += _byte_escapes[text_bytes[e.start]]
text_bytes = text_bytes[e.start+1:]
text_bytes = text_bytes[e.start + 1 :]
return ret
@ -51,9 +54,10 @@ def CEscape(text, as_utf8) -> str:
Args:
text: A byte string to be escaped.
as_utf8: Specifies if result may contain non-ASCII characters.
In Python 3 this allows unescaped non-ASCII Unicode characters.
In Python 2 the return value will be valid UTF-8 rather than only ASCII.
as_utf8: Specifies if result may contain non-ASCII characters. In Python 3
this allows unescaped non-ASCII Unicode characters. In Python 2 the return
value will be valid UTF-8 rather than only ASCII.
Returns:
Escaped string (str).
"""
@ -82,6 +86,7 @@ def CUnescape(text: str) -> bytes:
Args:
text: The data to parse in a str.
Returns:
A byte string.
"""

614
python/google/protobuf/text_format.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains routines for printing messages in Protobuf Text Format.
Printing and parsing messages in Text Format is useful for debugging
@ -41,17 +40,27 @@ import warnings
from google.protobuf.internal import decoder
from google.protobuf.internal import type_checkers
from google.protobuf import descriptor
from google.protobuf import text_encoding
from google.protobuf import unknown_fields
# pylint: disable=g-import-not-at-top
__all__ = ['MessageToString', 'Parse', 'PrintMessage', 'PrintField',
'PrintFieldValue', 'Merge', 'MessageToBytes']
__all__ = [
'MessageToString',
'Parse',
'PrintMessage',
'PrintField',
'PrintFieldValue',
'Merge',
'MessageToBytes',
]
_INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
type_checkers.Int32ValueChecker(),
type_checkers.Uint64ValueChecker(),
type_checkers.Int64ValueChecker())
_INTEGER_CHECKERS = (
type_checkers.Uint32ValueChecker(),
type_checkers.Int32ValueChecker(),
type_checkers.Uint64ValueChecker(),
type_checkers.Int64ValueChecker(),
)
_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?$', re.IGNORECASE)
_FLOAT_NAN = re.compile('nanf?$', re.IGNORECASE)
_FLOAT_OCTAL_PREFIX = re.compile('-?0[0-9]+')
@ -118,7 +127,8 @@ def MessageToString(
indent=0,
message_formatter=None,
print_unknown_fields=False,
force_colon=False) -> str:
force_colon=False,
) -> str:
"""Convert protobuf message to text format.
Args:
@ -180,9 +190,11 @@ def MessageToBytes(message, **kwargs) -> bytes:
def _IsMapEntry(field):
return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
field.message_type.has_options and
field.message_type.GetOptions().map_entry)
return (
field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
and field.message_type.has_options
and field.message_type.GetOptions().map_entry
)
def _IsGroupLike(field):
@ -215,19 +227,21 @@ def _IsGroupLike(field):
)
def PrintMessage(message,
out,
indent=0,
as_utf8=_as_utf8_default,
as_one_line=False,
use_short_repeated_primitives=False,
pointy_brackets=False,
use_index_order=False,
use_field_number=False,
descriptor_pool=None,
message_formatter=None,
print_unknown_fields=False,
force_colon=False):
def PrintMessage(
message,
out,
indent=0,
as_utf8=_as_utf8_default,
as_one_line=False,
use_short_repeated_primitives=False,
pointy_brackets=False,
use_index_order=False,
use_field_number=False,
descriptor_pool=None,
message_formatter=None,
print_unknown_fields=False,
force_colon=False,
):
"""Convert the message to text format and write it to the out stream.
Args:
@ -244,15 +258,17 @@ def PrintMessage(message,
field number order.
use_field_number: If True, print field numbers instead of names.
descriptor_pool: A DescriptorPool used to resolve Any types.
message_formatter: A function(message, indent, as_one_line): unicode|None
to custom format selected sub-messages (usually based on message type).
Use to pretty print parts of the protobuf for easier diffing.
message_formatter: A function(message, indent, as_one_line): unicode|None to
custom format selected sub-messages (usually based on message type). Use
to pretty print parts of the protobuf for easier diffing.
print_unknown_fields: If True, unknown fields will be printed.
force_colon: If set, a colon will be added after the field name even if
the field is a proto message.
force_colon: If set, a colon will be added after the field name even if the
field is a proto message.
"""
printer = _Printer(
out=out, indent=indent, as_utf8=as_utf8,
out=out,
indent=indent,
as_utf8=as_utf8,
as_one_line=as_one_line,
use_short_repeated_primitives=use_short_repeated_primitives,
pointy_brackets=pointy_brackets,
@ -261,22 +277,25 @@ def PrintMessage(message,
descriptor_pool=descriptor_pool,
message_formatter=message_formatter,
print_unknown_fields=print_unknown_fields,
force_colon=force_colon)
force_colon=force_colon,
)
printer.PrintMessage(message)
def PrintField(field,
value,
out,
indent=0,
as_utf8=_as_utf8_default,
as_one_line=False,
use_short_repeated_primitives=False,
pointy_brackets=False,
use_index_order=False,
message_formatter=None,
print_unknown_fields=False,
force_colon=False):
def PrintField(
field,
value,
out,
indent=0,
as_utf8=_as_utf8_default,
as_one_line=False,
use_short_repeated_primitives=False,
pointy_brackets=False,
use_index_order=False,
message_formatter=None,
print_unknown_fields=False,
force_colon=False,
):
"""Print a single field name/value pair."""
printer = _Printer(
out,
@ -293,18 +312,20 @@ def PrintField(field,
printer.PrintField(field, value)
def PrintFieldValue(field,
value,
out,
indent=0,
as_utf8=_as_utf8_default,
as_one_line=False,
use_short_repeated_primitives=False,
pointy_brackets=False,
use_index_order=False,
message_formatter=None,
print_unknown_fields=False,
force_colon=False):
def PrintFieldValue(
field,
value,
out,
indent=0,
as_utf8=_as_utf8_default,
as_one_line=False,
use_short_repeated_primitives=False,
pointy_brackets=False,
use_index_order=False,
message_formatter=None,
print_unknown_fields=False,
force_colon=False,
):
"""Print a single field value (not including name)."""
printer = _Printer(
out,
@ -409,8 +430,9 @@ class _Printer(object):
"""Serializes if message is a google.protobuf.Any field."""
if '/' not in message.type_url:
return False
packed_message = _BuildMessageFromTypeName(message.TypeName(),
self.descriptor_pool)
packed_message = _BuildMessageFromTypeName(
message.TypeName(), self.descriptor_pool
)
if packed_message is not None:
packed_message.MergeFromString(message.value)
colon = ':' if self.force_colon else ''
@ -440,13 +462,16 @@ class _Printer(object):
"""
if self.message_formatter and self._TryCustomFormatMessage(message):
return
if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and
self._TryPrintAsAnyMessage(message)):
if (
message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME
and self._TryPrintAsAnyMessage(message)
):
return
fields = message.ListFields()
if self.use_index_order:
fields.sort(
key=lambda x: x[0].number if x[0].is_extension else x[0].index)
key=lambda x: x[0].number if x[0].is_extension else x[0].index
)
for field, value in fields:
if _IsMapEntry(field):
for key in sorted(value):
@ -458,9 +483,11 @@ class _Printer(object):
entry_submsg = value.GetEntryClass()(key=key, value=value[key])
self.PrintField(field, entry_submsg)
elif field.is_repeated:
if (self.use_short_repeated_primitives
if (
self.use_short_repeated_primitives
and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE
and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_STRING):
and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_STRING
):
self._PrintShortRepeatedPrimitivesValue(field, value)
else:
for element in value:
@ -496,9 +523,10 @@ class _Printer(object):
# If this field is parseable as a Message, it is probably
# an embedded message.
# pylint: disable=protected-access
(embedded_unknown_message, pos) = decoder._DecodeUnknownFieldSet(
memoryview(field.data), 0, len(field.data))
except Exception: # pylint: disable=broad-except
embedded_unknown_message, pos = decoder._DecodeUnknownFieldSet(
memoryview(field.data), 0, len(field.data)
)
except Exception: # pylint: disable=broad-except
pos = 0
if pos == len(field.data):
@ -517,9 +545,9 @@ class _Printer(object):
out.write(' ' * self.indent + '}\n')
else:
# A string or bytes field. self.as_utf8 may not work.
out.write(': \"')
out.write(': "')
out.write(text_encoding.CEscape(field.data, False))
out.write('\" ' if self.as_one_line else '\"\n')
out.write('" ' if self.as_one_line else '"\n')
else:
# varint, fixed32, fixed64
out.write(': ')
@ -535,10 +563,12 @@ class _Printer(object):
else:
if field.is_extension:
out.write('[')
if (field.containing_type.GetOptions().message_set_wire_format and
field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
not field.is_required and
not field.is_repeated):
if (
field.containing_type.GetOptions().message_set_wire_format
and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
and not field.is_required
and not field.is_repeated
):
out.write(field.message_type.full_name)
else:
out.write(field.full_name)
@ -549,8 +579,10 @@ class _Printer(object):
else:
out.write(field.name)
if (self.force_colon or
field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE):
if (
self.force_colon
or field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE
):
# The colon is optional in this case, but our cross-language golden files
# don't include it. Here, the colon is only included if force_colon is
# set to True
@ -564,7 +596,7 @@ class _Printer(object):
self.out.write(' ' if self.as_one_line else '\n')
def _PrintShortRepeatedPrimitivesValue(self, field, value):
""""Prints short repeated primitives value."""
""" "Prints short repeated primitives value."""
# Note: this is called only when value has at least one element.
self._PrintFieldName(field)
self.out.write(' [')
@ -613,7 +645,7 @@ class _Printer(object):
else:
out.write(str(value))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
out.write('\"')
out.write('"')
if isinstance(value, str) and not self.as_utf8:
out_value = value.encode('utf-8')
else:
@ -624,7 +656,7 @@ class _Printer(object):
else:
out_as_utf8 = self.as_utf8
out.write(text_encoding.CEscape(out_value, out_as_utf8))
out.write('\"')
out.write('"')
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
if value:
out.write('true')
@ -639,13 +671,15 @@ class _Printer(object):
out.write(str(value))
def Parse(text,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None):
def Parse(
text,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None,
):
"""Parses a text representation of a protocol message into a message.
NOTE: for historical reasons this function does not clear the input
@ -680,19 +714,18 @@ def Parse(text,
parsing
allow_field_number: if True, both field number and field name are allowed.
descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.
allow_unknown_field: if True, skip over unknown field and keep
parsing. Avoid to use this option if possible. It may hide some
errors (e.g. spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the
message to be parsed: Text Format inputs over this depth will
fail to parse. ``None`` means no additional limit (the Python runtime
will enforce some limit due to call stack limits). As Text Format is
primarily intended to be used on trusted configuration inputs, and to
maintain backwards compatibility, the default of ``None`` (unbounded) is
intentional. For better consistency with what messages will successfully
round trip through binary wire format, or for the discouraged case of
processing untrusted Text Format inputs, setting a limit of 100 is
recommended.
allow_unknown_field: if True, skip over unknown field and keep parsing.
Avoid to use this option if possible. It may hide some errors (e.g.
spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the message to be
parsed: Text Format inputs over this depth will fail to parse. ``None``
means no additional limit (the Python runtime will enforce some limit
due to call stack limits). As Text Format is primarily intended to be
used on trusted configuration inputs, and to maintain backwards
compatibility, the default of ``None`` (unbounded) is intentional. For
better consistency with what messages will successfully round trip
through binary wire format, or for the discouraged case of processing
untrusted Text Format inputs, setting a limit of 100 is recommended.
Returns:
Message: The same message passed as argument.
@ -700,22 +733,26 @@ def Parse(text,
Raises:
ParseError: On text parsing problems.
"""
return ParseLines(text.split(b'\n' if isinstance(text, bytes) else u'\n'),
message,
allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth)
return ParseLines(
text.split(b'\n' if isinstance(text, bytes) else '\n'),
message,
allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth,
)
def Merge(text,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None):
def Merge(
text,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None,
):
"""Parses a text representation of a protocol message into a message.
Like Parse(), but allows repeated values for a non-repeated field, and uses
@ -729,19 +766,19 @@ def Merge(text,
parsing
allow_field_number: if True, both field number and field name are allowed.
descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.
allow_unknown_field: if True, skip over unknown field and keep
parsing. Avoid to use this option if possible. It may hide some
errors (e.g. spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the
message to be parsed: Text Format inputs over this depth will
fail to parse. ``None`` means no additional limit (the Python runtime
will enforce some limit due to call stack limits). As Text Format is
primarily intended to be used on trusted configuration inputs, and to
maintain backwards compatibility, the default of ``None`` (unbounded) is
intentional. For better consistency with what messages will successfully
round trip through binary wire format, or for the discouraged case of
processing untrusted Text Format inputs, setting a limit of 100 is
recommended.
allow_unknown_field: if True, skip over unknown field and keep parsing.
Avoid to use this option if possible. It may hide some errors (e.g.
spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the message to be
parsed: Text Format inputs over this depth will fail to parse. ``None``
means no additional limit (the Python runtime will enforce some limit
due to call stack limits). As Text Format is primarily intended to be
used on trusted configuration inputs, and to maintain backwards
compatibility, the default of ``None`` (unbounded) is intentional. For
better consistency with what messages will successfully round trip
through binary wire format, or for the discouraged case of processing
untrusted Text Format inputs, setting a limit of 100 is recommended.
Returns:
Message: The same message passed as argument.
@ -749,22 +786,25 @@ def Merge(text,
ParseError: On text parsing problems.
"""
return MergeLines(
text.split(b'\n' if isinstance(text, bytes) else u'\n'),
text.split(b'\n' if isinstance(text, bytes) else '\n'),
message,
allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth)
max_recursion_depth=max_recursion_depth,
)
def ParseLines(lines,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None):
def ParseLines(
lines,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None,
):
"""Parses a text representation of a protocol message into a message.
See Parse() for caveats.
@ -776,19 +816,18 @@ def ParseLines(lines,
parsing
allow_field_number: if True, both field number and field name are allowed.
descriptor_pool: A DescriptorPool used to resolve Any types.
allow_unknown_field: if True, skip over unknown field and keep
parsing. Avoid to use this option if possible. It may hide some
errors (e.g. spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the
message to be parsed: Text Format inputs over this depth will
fail to parse. ``None`` means no additional limit (the Python runtime
will enforce some limit due to call stack limits). As Text Format is
primarily intended to be used on trusted configuration inputs, and to
maintain backwards compatibility, the default of ``None`` (unbounded) is
intentional. For better consistency with what messages will successfully
round trip through binary wire format, or for the discouraged case of
processing untrusted Text Format inputs, setting a limit of 100 is
recommended.
allow_unknown_field: if True, skip over unknown field and keep parsing.
Avoid to use this option if possible. It may hide some errors (e.g.
spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the message to be
parsed: Text Format inputs over this depth will fail to parse. ``None``
means no additional limit (the Python runtime will enforce some limit
due to call stack limits). As Text Format is primarily intended to be
used on trusted configuration inputs, and to maintain backwards
compatibility, the default of ``None`` (unbounded) is intentional. For
better consistency with what messages will successfully round trip
through binary wire format, or for the discouraged case of processing
untrusted Text Format inputs, setting a limit of 100 is recommended.
Returns:
The same message passed as argument.
@ -796,21 +835,25 @@ def ParseLines(lines,
Raises:
ParseError: On text parsing problems.
"""
parser = _Parser(allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth)
parser = _Parser(
allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth,
)
return parser.ParseLines(lines, message)
def MergeLines(lines,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None):
def MergeLines(
lines,
message,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None,
):
"""Parses a text representation of a protocol message into a message.
See Merge() for more details.
@ -822,19 +865,18 @@ def MergeLines(lines,
parsing
allow_field_number: if True, both field number and field name are allowed.
descriptor_pool: A DescriptorPool used to resolve Any types.
allow_unknown_field: if True, skip over unknown field and keep
parsing. Avoid to use this option if possible. It may hide some
errors (e.g. spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the
message to be parsed: Text Format inputs over this depth will
fail to parse. ``None`` means no additional limit (the Python runtime
will enforce some limit due to call stack limits). As Text Format is
primarily intended to be used on trusted configuration inputs, and to
maintain backwards compatibility, the default of ``None`` (unbounded) is
intentional. For better consistency with what messages will successfully
round trip through binary wire format, or for the discouraged case of
processing untrusted Text Format inputs, setting a limit of 100 is
recommended.
allow_unknown_field: if True, skip over unknown field and keep parsing.
Avoid to use this option if possible. It may hide some errors (e.g.
spelling error on field name)
max_recursion_depth: Optional maximum recursion depth of the message to be
parsed: Text Format inputs over this depth will fail to parse. ``None``
means no additional limit (the Python runtime will enforce some limit
due to call stack limits). As Text Format is primarily intended to be
used on trusted configuration inputs, and to maintain backwards
compatibility, the default of ``None`` (unbounded) is intentional. For
better consistency with what messages will successfully round trip
through binary wire format, or for the discouraged case of processing
untrusted Text Format inputs, setting a limit of 100 is recommended.
Returns:
The same message passed as argument.
@ -842,23 +884,27 @@ def MergeLines(lines,
Raises:
ParseError: On text parsing problems.
"""
parser = _Parser(allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth)
parser = _Parser(
allow_unknown_extension,
allow_field_number,
descriptor_pool=descriptor_pool,
allow_unknown_field=allow_unknown_field,
max_recursion_depth=max_recursion_depth,
)
return parser.MergeLines(lines, message)
class _Parser(object):
"""Text format parser for protocol message."""
def __init__(self,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None):
def __init__(
self,
allow_unknown_extension=False,
allow_field_number=False,
descriptor_pool=None,
allow_unknown_field=False,
max_recursion_depth=None,
):
self.allow_unknown_extension = allow_unknown_extension
self.allow_field_number = allow_field_number
self.descriptor_pool = descriptor_pool
@ -892,7 +938,8 @@ class _Parser(object):
try:
str_lines = (
line if isinstance(line, str) else line.decode('utf-8')
for line in lines)
for line in lines
)
tokenizer = Tokenizer(str_lines)
except UnicodeDecodeError as e:
raise ParseError from e
@ -925,9 +972,7 @@ class _Parser(object):
)
while not tokenizer.TryConsume(end_token):
if tokenizer.AtEnd():
raise tokenizer.ParseErrorPreviousToken(
'Expected "%s".' % (end_token,)
)
raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,))
self._MergeField(tokenizer, message)
self.recursion_depth -= 1
@ -942,8 +987,10 @@ class _Parser(object):
ParseError: In case of text parsing problems.
"""
message_descriptor = message.DESCRIPTOR
if (message_descriptor.full_name == _ANY_FULL_TYPE_NAME and
tokenizer.TryConsume('[')):
if (
message_descriptor.full_name == _ANY_FULL_TYPE_NAME
and tokenizer.TryConsume('[')
):
type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
tokenizer.TryConsume(':')
self._DetectSilentMarker(
@ -956,14 +1003,16 @@ class _Parser(object):
else:
tokenizer.Consume('{')
expanded_any_end_token = '}'
expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,
self.descriptor_pool)
expanded_any_sub_message = _BuildMessageFromTypeName(
packed_type_name, self.descriptor_pool
)
# Direct comparison with None is used instead of implicit bool conversion
# to avoid false positives with falsy initial values, e.g. for
# google.protobuf.ListValue.
if expanded_any_sub_message is None:
raise ParseError('Type %s not found in descriptor pool' %
packed_type_name)
raise ParseError(
'Type %s not found in descriptor pool' % packed_type_name
)
self._MergeMessage(
tokenizer, expanded_any_sub_message, expanded_any_end_token
)
@ -984,8 +1033,9 @@ class _Parser(object):
if not message_descriptor.is_extendable:
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" does not have extensions.' %
message_descriptor.full_name)
'Message type "%s" does not have extensions.'
% message_descriptor.full_name
)
# pylint: disable=protected-access
field = message.Extensions._FindExtensionByName(name)
# pylint: enable=protected-access
@ -998,11 +1048,13 @@ class _Parser(object):
'Did you import the _pb2 module which defines it? '
'If you are trying to place the extension in the MessageSet '
'field of another message that is in an Any or MessageSet field, '
'that message\'s _pb2 module must be imported as well' % name)
"that message's _pb2 module must be imported as well" % name
)
elif message_descriptor != field.containing_type:
raise tokenizer.ParseErrorPreviousToken(
'Extension "%s" does not extend message type "%s".' %
(name, message_descriptor.full_name))
'Extension "%s" does not extend message type "%s".'
% (name, message_descriptor.full_name)
)
tokenizer.Consume(']')
@ -1028,8 +1080,9 @@ class _Parser(object):
if not field and not self.allow_unknown_field:
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" has no field named "%s".' %
(message_descriptor.full_name, name))
'Message type "%s" has no field named "%s".'
% (message_descriptor.full_name, name)
)
if field:
if not self._allow_multiple_scalars and field.containing_oneof:
@ -1040,23 +1093,29 @@ class _Parser(object):
if which_oneof is not None and which_oneof != field.name:
raise tokenizer.ParseErrorPreviousToken(
'Field "%s" is specified along with field "%s", another member '
'of oneof "%s" for message type "%s".' %
(field.name, which_oneof, field.containing_oneof.name,
message_descriptor.full_name))
'of oneof "%s" for message type "%s".'
% (
field.name,
which_oneof,
field.containing_oneof.name,
message_descriptor.full_name,
)
)
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
tokenizer.TryConsume(':')
self._DetectSilentMarker(tokenizer, message_descriptor.full_name,
field.full_name)
self._DetectSilentMarker(
tokenizer, message_descriptor.full_name, field.full_name
)
merger = self._MergeMessageField
else:
tokenizer.Consume(':')
self._DetectSilentMarker(tokenizer, message_descriptor.full_name,
field.full_name)
self._DetectSilentMarker(
tokenizer, message_descriptor.full_name, field.full_name
)
merger = self._MergeScalarField
if (field.is_repeated and
tokenizer.TryConsume('[')):
if field.is_repeated and tokenizer.TryConsume('['):
# Short repeated format, e.g. "foo: [1, 2, 3]"
if not tokenizer.TryConsume(']'):
while True:
@ -1069,7 +1128,7 @@ class _Parser(object):
merger(tokenizer, message, field)
else: # Proto field is unknown.
assert (self.allow_unknown_extension or self.allow_unknown_field)
assert self.allow_unknown_extension or self.allow_unknown_field
self._SkipFieldContents(tokenizer, name, message_descriptor.full_name)
# For historical reasons, fields may optionally be separated by commas or
@ -1167,20 +1226,20 @@ class _Parser(object):
sub_message = getattr(message, field.name).add()
else:
if field.is_extension:
if (not self._allow_multiple_scalars and
message.HasExtension(field)):
if not self._allow_multiple_scalars and message.HasExtension(field):
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" should not have multiple "%s" extensions.' %
(message.DESCRIPTOR.full_name, field.full_name))
'Message type "%s" should not have multiple "%s" extensions.'
% (message.DESCRIPTOR.full_name, field.full_name)
)
sub_message = message.Extensions[field]
else:
# Also apply _allow_multiple_scalars to message field.
# TODO: Change to _allow_singular_overwrites.
if (not self._allow_multiple_scalars and
message.HasField(field.name)):
if not self._allow_multiple_scalars and message.HasField(field.name):
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" should not have multiple "%s" fields.' %
(message.DESCRIPTOR.full_name, field.name))
'Message type "%s" should not have multiple "%s" fields.'
% (message.DESCRIPTOR.full_name, field.name)
)
sub_message = getattr(message, field.name)
sub_message.SetInParent()
@ -1209,22 +1268,32 @@ class _Parser(object):
_ = self.allow_unknown_extension
value = None
if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
descriptor.FieldDescriptor.TYPE_SINT32,
descriptor.FieldDescriptor.TYPE_SFIXED32):
if field.type in (
descriptor.FieldDescriptor.TYPE_INT32,
descriptor.FieldDescriptor.TYPE_SINT32,
descriptor.FieldDescriptor.TYPE_SFIXED32,
):
value = _ConsumeInt32(tokenizer)
elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
descriptor.FieldDescriptor.TYPE_SINT64,
descriptor.FieldDescriptor.TYPE_SFIXED64):
elif field.type in (
descriptor.FieldDescriptor.TYPE_INT64,
descriptor.FieldDescriptor.TYPE_SINT64,
descriptor.FieldDescriptor.TYPE_SFIXED64,
):
value = _ConsumeInt64(tokenizer)
elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
descriptor.FieldDescriptor.TYPE_FIXED32):
elif field.type in (
descriptor.FieldDescriptor.TYPE_UINT32,
descriptor.FieldDescriptor.TYPE_FIXED32,
):
value = _ConsumeUint32(tokenizer)
elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
descriptor.FieldDescriptor.TYPE_FIXED64):
elif field.type in (
descriptor.FieldDescriptor.TYPE_UINT64,
descriptor.FieldDescriptor.TYPE_FIXED64,
):
value = _ConsumeUint64(tokenizer)
elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
descriptor.FieldDescriptor.TYPE_DOUBLE):
elif field.type in (
descriptor.FieldDescriptor.TYPE_FLOAT,
descriptor.FieldDescriptor.TYPE_DOUBLE,
):
value = tokenizer.ConsumeFloat()
elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
value = tokenizer.ConsumeBool()
@ -1244,12 +1313,15 @@ class _Parser(object):
getattr(message, field.name).append(value)
else:
if field.is_extension:
if (not self._allow_multiple_scalars and
field.has_presence and
message.HasExtension(field)):
if (
not self._allow_multiple_scalars
and field.has_presence
and message.HasExtension(field)
):
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" should not have multiple "%s" extensions.' %
(message.DESCRIPTOR.full_name, field.full_name))
'Message type "%s" should not have multiple "%s" extensions.'
% (message.DESCRIPTOR.full_name, field.full_name)
)
else:
message.Extensions[field] = value
else:
@ -1266,8 +1338,9 @@ class _Parser(object):
if duplicate_error:
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" should not have multiple "%s" fields.' %
(message.DESCRIPTOR.full_name, field.name))
'Message type "%s" should not have multiple "%s" fields.'
% (message.DESCRIPTOR.full_name, field.name)
)
else:
setattr(message, field.name, value)
@ -1277,8 +1350,8 @@ class _Parser(object):
Args:
tokenizer: A tokenizer to parse the field name and values.
field_name: The field name currently being parsed.
immediate_message_type: The type of the message immediately containing
the silent marker.
immediate_message_type: The type of the message immediately containing the
silent marker.
"""
# Try to guess the type of this field.
# If this field is not a message, there should be a ":" between the
@ -1286,8 +1359,11 @@ class _Parser(object):
# start with "{" or "<" which indicates the beginning of a message body.
# If there is no ":" or there is a "{" or "<" after ":", this field has
# to be a message or the input is ill-formed.
if tokenizer.TryConsume(
':') and not tokenizer.LookingAt('{') and not tokenizer.LookingAt('<'):
if (
tokenizer.TryConsume(':')
and not tokenizer.LookingAt('{')
and not tokenizer.LookingAt('<')
):
self._DetectSilentMarker(tokenizer, immediate_message_type, field_name)
if tokenizer.LookingAt('['):
self._SkipRepeatedFieldValue(tokenizer, immediate_message_type)
@ -1302,8 +1378,8 @@ class _Parser(object):
Args:
tokenizer: A tokenizer to parse the field name and values.
immediate_message_type: The type of the message immediately containing
the silent marker.
immediate_message_type: The type of the message immediately containing the
silent marker.
"""
field_name = ''
if tokenizer.TryConsume('['):
@ -1335,8 +1411,8 @@ class _Parser(object):
Args:
tokenizer: A tokenizer to parse the field name and values.
immediate_message_type: The type of the message immediately containing
the silent marker
immediate_message_type: The type of the message immediately containing the
silent marker
"""
if tokenizer.TryConsume('<'):
delimiter = '>'
@ -1358,11 +1434,13 @@ class _Parser(object):
Raises:
ParseError: In case an invalid field value is found.
"""
if (not tokenizer.TryConsumeByteString()and
not tokenizer.TryConsumeIdentifier() and
not _TryConsumeInt64(tokenizer) and
not _TryConsumeUint64(tokenizer) and
not tokenizer.TryConsumeFloat()):
if (
not tokenizer.TryConsumeByteString()
and not tokenizer.TryConsumeIdentifier()
and not _TryConsumeInt64(tokenizer)
and not _TryConsumeUint64(tokenizer)
and not tokenizer.TryConsumeFloat()
):
raise ParseError('Invalid field value: ' + tokenizer.token)
def _SkipRepeatedFieldValue(self, tokenizer, immediate_message_type):
@ -1428,8 +1506,9 @@ class Tokenizer(object):
self._previous_column = 0
self._more_lines = True
self._skip_comments = skip_comments
self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT
or self._WHITESPACE)
self._whitespace_pattern = (
skip_comments and self._WHITESPACE_OR_COMMENT or self._WHITESPACE
)
self.contains_silent_marker_before_current_token = False
self._SkipWhitespace()
@ -1465,7 +1544,8 @@ class Tokenizer(object):
if not match:
break
self.contains_silent_marker_before_current_token = match.group(0) == (
' ' + _DEBUG_STRING_SILENT_MARKER)
' ' + _DEBUG_STRING_SILENT_MARKER
)
length = len(match.group(0))
self._column += length
@ -1513,8 +1593,7 @@ class Tokenizer(object):
comment = self.ConsumeComment()
# A trailing comment is a comment on the same line than the previous token.
trailing = (self._previous_line == before_parsing
and not just_started)
trailing = self._previous_line == before_parsing and not just_started
return trailing, comment
@ -1728,16 +1807,20 @@ class Tokenizer(object):
Returns:
A ParseError instance.
"""
return ParseError(message, self._previous_line + 1,
self._previous_column + 1)
return ParseError(
message, self._previous_line + 1, self._previous_column + 1
)
def ParseError(self, message):
"""Creates and *returns* a ParseError for the current token."""
return ParseError('\'' + self._current_line + '\': ' + message,
self._line + 1, self._column + 1)
return ParseError(
"'" + self._current_line + "': " + message,
self._line + 1,
self._column + 1,
)
def _StringParseError(self, e):
return self.ParseError('Couldn\'t parse string: ' + str(e))
return self.ParseError("Couldn't parse string: " + str(e))
def NextToken(self):
"""Reads the next meaningful token."""
@ -1761,6 +1844,7 @@ class Tokenizer(object):
else:
self.token = self._current_line[self._column]
# Aliased so it can still be accessed by current visibility violators.
# TODO: Migrate violators to textformat_tokenizer.
_Tokenizer = Tokenizer # pylint: disable=invalid-name
@ -1909,7 +1993,7 @@ def _ParseAbstractInteger(text):
try:
return int(text, 0)
except ValueError:
raise ValueError('Couldn\'t parse integer: %s' % orig_text)
raise ValueError("Couldn't parse integer: %s" % orig_text)
def ParseFloat(text):
@ -1989,13 +2073,17 @@ def ParseEnum(field, value):
# Identifier.
enum_value = enum_descriptor.values_by_name.get(value, None)
if enum_value is None:
raise ValueError('Enum type "%s" has no value named %s.' %
(enum_descriptor.full_name, value))
raise ValueError(
'Enum type "%s" has no value named %s.'
% (enum_descriptor.full_name, value)
)
else:
if not field.enum_type.is_closed:
return number
enum_value = enum_descriptor.values_by_number.get(number, None)
if enum_value is None:
raise ValueError('Enum type "%s" has no value with number %d.' %
(enum_descriptor.full_name, number))
raise ValueError(
'Enum type "%s" has no value with number %d.'
% (enum_descriptor.full_name, number)
)
return enum_value.number

9
python/google/protobuf/unknown_fields.py
View file

@ -4,7 +4,6 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains Unknown Fields APIs.
Simple usage example:
@ -15,7 +14,6 @@ Simple usage example:
data = unknown_field.data
"""
from google.protobuf.internal import api_implementation
if api_implementation._c_module is not None: # pylint: disable=protected-access
@ -64,18 +62,17 @@ else:
msg_des = msg.DESCRIPTOR
# pylint: disable=protected-access
unknown_fields = msg._unknown_fields
if (msg_des.has_options and
msg_des.GetOptions().message_set_wire_format):
if msg_des.has_options and msg_des.GetOptions().message_set_wire_format:
local_decoder = decoder.UnknownMessageSetItemDecoder()
for _, buffer in unknown_fields:
(field_number, data) = local_decoder(memoryview(buffer))
field_number, data = local_decoder(memoryview(buffer))
InternalAdd(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED, data)
else:
for tag_bytes, buffer in unknown_fields:
field_number, wire_type = decoder.DecodeTag(tag_bytes)
if field_number == 0:
raise RuntimeError('Field number 0 is illegal.')
(data, _) = decoder._DecodeUnknownField(
data, _ = decoder._DecodeUnknownField(
memoryview(buffer), 0, len(buffer), field_number, wire_type
)
InternalAdd(field_number, wire_type, data)

174
python/minimal_test.py
View file

@ -30,99 +30,104 @@
"""A bare-bones unit test that doesn't load any generated code."""
import unittest
from google.protobuf.pyext import _message
from google3.net.proto2.python.internal import api_implementation
from google.protobuf import unittest_pb2
from google.protobuf import map_unittest_pb2
from google.protobuf import descriptor_pool
from google.protobuf import text_format
from google.protobuf import message_factory
from google.protobuf import message
from google3.net.proto2.python.internal import factory_test1_pb2
from google3.net.proto2.python.internal import factory_test2_pb2
from google3.net.proto2.python.internal import more_extensions_pb2
from google.protobuf import descriptor_pb2
from google.protobuf import descriptor_pool
from google.protobuf import map_unittest_pb2
from google.protobuf import message
from google.protobuf import message_factory
from google.protobuf import text_format
from google.protobuf import unittest_pb2
class TestMessageExtension(unittest.TestCase):
def test_descriptor_pool(self):
serialized_desc = b'\n\ntest.proto\"\x0e\n\x02M1*\x08\x08\x01\x10\x80\x80\x80\x80\x02:\x15\n\x08test_ext\x12\x03.M1\x18\x01 \x01(\x05'
pool = _message.DescriptorPool()
file_desc = pool.AddSerializedFile(serialized_desc)
self.assertEqual("test.proto", file_desc.name)
ext_desc = pool.FindExtensionByName("test_ext")
self.assertEqual(1, ext_desc.number)
def test_descriptor_pool(self):
serialized_desc = (
b'\n\ntest.proto"\x0e\n\x02M1*\x08\x08\x01\x10\x80\x80\x80\x80\x02:\x15\n\x08test_ext\x12\x03.M1\x18\x01'
b" \x01(\x05"
)
pool = _message.DescriptorPool()
file_desc = pool.AddSerializedFile(serialized_desc)
self.assertEqual("test.proto", file_desc.name)
ext_desc = pool.FindExtensionByName("test_ext")
self.assertEqual(1, ext_desc.number)
# Test object cache: repeatedly retrieving the same descriptor
# should result in the same object
self.assertIs(ext_desc, pool.FindExtensionByName("test_ext"))
# Test object cache: repeatedly retrieving the same descriptor
# should result in the same object
self.assertIs(ext_desc, pool.FindExtensionByName("test_ext"))
def test_lib_is_upb(self):
# Ensure we are not pulling in a different protobuf library on the
# system.
print(_message._IS_UPB)
self.assertTrue(_message._IS_UPB)
self.assertEqual(api_implementation.Type(), "cpp")
def test_lib_is_upb(self):
# Ensure we are not pulling in a different protobuf library on the
# system.
print(_message._IS_UPB)
self.assertTrue(_message._IS_UPB)
self.assertEqual(api_implementation.Type(), "cpp")
def test_repeated_field_slice_delete(self):
def test_slice(start, end, step):
vals = list(range(20))
message = unittest_pb2.TestAllTypes(repeated_int32=vals)
del vals[start:end:step]
del message.repeated_int32[start:end:step]
self.assertEqual(vals, list(message.repeated_int32))
def test_repeated_field_slice_delete(self):
def test_slice(start, end, step):
vals = list(range(20))
message = unittest_pb2.TestAllTypes(repeated_int32=vals)
del vals[start:end:step]
del message.repeated_int32[start:end:step]
self.assertEqual(vals, list(message.repeated_int32))
test_slice(3, 11, 1)
test_slice(3, 11, 2)
test_slice(3, 11, 3)
test_slice(11, 3, -1)
test_slice(11, 3, -2)
test_slice(11, 3, -3)
test_slice(10, 25, 4)
def testExtensionsErrors(self):
msg = unittest_pb2.TestAllTypes()
self.assertRaises(AttributeError, getattr, msg, 'Extensions')
def testClearStubMapField(self):
msg = map_unittest_pb2.TestMapSubmessage()
int32_map = msg.test_map.map_int32_int32
msg.test_map.ClearField("map_int32_int32")
int32_map[123] = 456
self.assertEqual(0, msg.test_map.ByteSize())
test_slice(3, 11, 1)
test_slice(3, 11, 2)
test_slice(3, 11, 3)
test_slice(11, 3, -1)
test_slice(11, 3, -2)
test_slice(11, 3, -3)
test_slice(10, 25, 4)
def testClearReifiedMapField(self):
msg = map_unittest_pb2.TestMap()
int32_map = msg.map_int32_int32
int32_map[123] = 456
msg.ClearField("map_int32_int32")
int32_map[111] = 222
self.assertEqual(0, msg.ByteSize())
def testExtensionsErrors(self):
msg = unittest_pb2.TestAllTypes()
self.assertRaises(AttributeError, getattr, msg, "Extensions")
def testClearStubRepeatedField(self):
msg = unittest_pb2.NestedTestAllTypes()
int32_array = msg.payload.repeated_int32
msg.payload.ClearField("repeated_int32")
int32_array.append(123)
self.assertEqual(0, msg.payload.ByteSize())
def testClearStubMapField(self):
msg = map_unittest_pb2.TestMapSubmessage()
int32_map = msg.test_map.map_int32_int32
msg.test_map.ClearField("map_int32_int32")
int32_map[123] = 456
self.assertEqual(0, msg.test_map.ByteSize())
def testClearReifiedRepeatdField(self):
msg = unittest_pb2.TestAllTypes()
int32_array = msg.repeated_int32
int32_array.append(123)
self.assertNotEqual(0, msg.ByteSize())
msg.ClearField("repeated_int32")
int32_array.append(123)
self.assertEqual(0, msg.ByteSize())
def testClearReifiedMapField(self):
msg = map_unittest_pb2.TestMap()
int32_map = msg.map_int32_int32
int32_map[123] = 456
msg.ClearField("map_int32_int32")
int32_map[111] = 222
self.assertEqual(0, msg.ByteSize())
def testClearStubRepeatedField(self):
msg = unittest_pb2.NestedTestAllTypes()
int32_array = msg.payload.repeated_int32
msg.payload.ClearField("repeated_int32")
int32_array.append(123)
self.assertEqual(0, msg.payload.ByteSize())
def testClearReifiedRepeatdField(self):
msg = unittest_pb2.TestAllTypes()
int32_array = msg.repeated_int32
int32_array.append(123)
self.assertNotEqual(0, msg.ByteSize())
msg.ClearField("repeated_int32")
int32_array.append(123)
self.assertEqual(0, msg.ByteSize())
def testFloatPrinting(self):
message = unittest_pb2.TestAllTypes()
message.optional_float = -0.0
self.assertEqual(str(message), "optional_float: -0\n")
def testFloatPrinting(self):
message = unittest_pb2.TestAllTypes()
message.optional_float = -0.0
self.assertEqual(str(message), 'optional_float: -0\n')
class OversizeProtosTest(unittest.TestCase):
def setUp(self):
msg = unittest_pb2.NestedTestAllTypes()
m = msg
@ -130,17 +135,19 @@ class OversizeProtosTest(unittest.TestCase):
m = m.child
m.Clear()
self.p_serialized = msg.SerializeToString()
def testAssertOversizeProto(self):
from google.protobuf.pyext._message import SetAllowOversizeProtos
SetAllowOversizeProtos(False)
q = unittest_pb2.NestedTestAllTypes()
with self.assertRaises(message.DecodeError):
q.ParseFromString(self.p_serialized)
print(q)
def testSucceedOversizeProto(self):
from google.protobuf.pyext._message import SetAllowOversizeProtos
SetAllowOversizeProtos(True)
q = unittest_pb2.NestedTestAllTypes()
q.ParseFromString(self.p_serialized)
@ -159,12 +166,12 @@ class OversizeProtosTest(unittest.TestCase):
# Set some normal fields.
extendee_proto.optional_int32 = 1
extendee_proto.repeated_string.append('hi')
extendee_proto.repeated_string.append("hi")
expected = {
extension_int32: True,
extension_msg: True,
extension_repeated: True
extension_repeated: True,
}
count = 0
for item in extendee_proto.Extensions:
@ -173,15 +180,18 @@ class OversizeProtosTest(unittest.TestCase):
count += 1
self.assertEqual(count, 3)
self.assertEqual(len(expected), 0)
def testIsInitializedStub(self):
proto = unittest_pb2.TestRequiredForeign()
self.assertTrue(proto.IsInitialized())
self.assertFalse(proto.optional_message.IsInitialized())
errors = []
self.assertFalse(proto.optional_message.IsInitialized(errors))
self.assertEqual(['a', 'b', 'c'], errors)
self.assertRaises(message.EncodeError, proto.optional_message.SerializeToString)
self.assertEqual(["a", "b", "c"], errors)
self.assertRaises(
message.EncodeError, proto.optional_message.SerializeToString
)
if __name__ == '__main__':
unittest.main(verbosity=2)
if __name__ == "__main__":
unittest.main(verbosity=2)