mmo-ref/dep-protobuf
1
0
Fork 0
mirror of https://github.com/protocolbuffers/protobuf synced 2026-08-26 02:23:14 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #29424 from protocolbuffers/tonyliaoss-python-cherrypick

Browse source 
python cherrypick of custom JSON name parser cache and additional unit tests.
This commit is contained in:
Tony Liao 2026-08-25 15:57:07 -04:00 • committed by GitHub
commit 298256df8f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 511 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

250
python/google/protobuf/internal/json_format_benchmark.py Normal file
View file

@ -0,0 +1,250 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2026 Google LLC. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Microbenchmarks for JSON format parsing and custom enum names."""
import statistics
import timeit
import unittest
from google.protobuf import json_format
from google.protobuf.json import json_enumval_custom_string_pb2
class JsonFormatBenchmark(unittest.TestCase):
"""Microbenchmarks for JSON format parsing."""
def test_benchmark_parse_json(self):
"""Benchmarks parsing of default, custom, and unknown enum names in JSON."""
default_payloads = [
'{"armor":"ARMOR_GREAT_HELM"}',
'{"armor":"ARMOR_GAUNTLET"}',
'{"armor":"ARMOR_PLATE"}',
'{"armor":"ARMOR_COIF"}',
'{"armor":"ARMOR_PAULDRON"}',
'{"armor":"ARMOR_SABATON"}',
'{"armor":"ARMOR_HACHI_MAI_DO"}',
]
custom_payloads = [
'{"armor":"gr8 helm"}',
'{"armor":"a\\"b"}',
'{"armor":"\\"plate\\""}',
'{"armor":""}',
'{"armor":"p\\taul\\ndron"}',
'{"armor":"sabaton"}',
'{"armor":"8"}',
]
unknown_payloads = [
'{"armor":"UNKNOWN_1"}',
'{"armor":"UNKNOWN_2"}',
'{"armor":"UNKNOWN_3"}',
'{"armor":"UNKNOWN_4"}',
'{"armor":"UNKNOWN_5"}',
'{"armor":"UNKNOWN_6"}',
'{"armor":"UNKNOWN_7"}',
]
repeated_default_payload = (
'{"armors":['
+ ','.join(
[
'"ARMOR_GREAT_HELM"',
'"ARMOR_GAUNTLET"',
'"ARMOR_PLATE"',
'"ARMOR_COIF"',
'"ARMOR_PAULDRON"',
'"ARMOR_SABATON"',
'"ARMOR_HACHI_MAI_DO"',
]
* 14
)
+ ']}'
)
repeated_custom_payload = (
'{"armors":['
+ ','.join(
[
'"gr8 helm"',
'"a\\"b"',
'"\\"plate\\""',
'""',
'"p\\taul\\ndron"',
'"sabaton"',
'"8"',
]
* 14
)
+ ']}'
)
repeated_unknown_payload = (
'{"armors":['
+ ','.join(
[
'"UNKNOWN_1"',
'"UNKNOWN_2"',
'"UNKNOWN_3"',
'"UNKNOWN_4"',
'"UNKNOWN_5"',
'"UNKNOWN_6"',
'"UNKNOWN_7"',
]
* 14
)
+ ']}'
)
iterations = 10
trials = 100
num_parses = 10
num_items = 98
msg = json_enumval_custom_string_pb2.Knight()
# Warmup phase to eliminate initial compilation/import overhead
for _ in range(200):
for p in custom_payloads:
msg.Clear()
json_format.Parse(p, msg)
def run_def():
for p in default_payloads:
for _ in range(num_parses):
msg.Clear()
json_format.Parse(p, msg)
def run_cust():
for p in custom_payloads:
for _ in range(num_parses):
msg.Clear()
json_format.Parse(p, msg)
def run_unk():
for p in unknown_payloads:
for _ in range(num_parses):
msg.Clear()
json_format.Parse(p, msg, ignore_unknown_fields=True)
def run_rep_def():
for _ in range(num_parses):
msg.Clear()
json_format.Parse(repeated_default_payload, msg)
def run_rep_cust():
for _ in range(num_parses):
msg.Clear()
json_format.Parse(repeated_custom_payload, msg)
def run_rep_unk():
for _ in range(num_parses):
msg.Clear()
json_format.Parse(
repeated_unknown_payload, msg, ignore_unknown_fields=True
)
benchmarks = [
(
'run_def',
timeit.Timer(run_def, setup=msg.Clear),
len(default_payloads) * num_parses,
),
(
'run_cust',
timeit.Timer(run_cust, setup=msg.Clear),
len(custom_payloads) * num_parses,
),
(
'run_unk',
timeit.Timer(run_unk, setup=msg.Clear),
len(unknown_payloads) * num_parses,
),
(
'run_rep_def',
timeit.Timer(run_rep_def, setup=msg.Clear),
num_items * num_parses,
),
(
'run_rep_cust',
timeit.Timer(run_rep_cust, setup=msg.Clear),
num_items * num_parses,
),
(
'run_rep_unk',
timeit.Timer(run_rep_unk, setup=msg.Clear),
num_items * num_parses,
),
]
# Interleaved trials to minimize CPU frequency scaling / ordering bias
benchmark_samples = {name: [] for name, _, _ in benchmarks}
for _ in range(trials):
for name, timer, num_ops_per_batch in benchmarks:
total_ops = iterations * num_ops_per_batch
elapsed = timer.timeit(number=iterations)
benchmark_samples[name].append(elapsed * 1e6 / total_ops)
def stats(samples):
if len(samples) < 100:
raise ValueError(
f'Insufficient samples ({len(samples)}) for statistical'
' calculations. At least 100 samples are required.'
)
med_val = statistics.median(samples)
mean_val = statistics.mean(samples)
stdev_val = statistics.stdev(samples)
p99_val = statistics.quantiles(samples, n=100)[98]
return p99_val, med_val, mean_val, stdev_val
p99_def, med_def, mean_def, std_def = stats(benchmark_samples['run_def'])
p99_cust, med_cust, mean_cust, std_cust = stats(
benchmark_samples['run_cust']
)
p99_unk, med_unk, mean_unk, std_unk = stats(benchmark_samples['run_unk'])
p99_rep_def, med_rep_def, mean_rep_def, std_rep_def = stats(
benchmark_samples['run_rep_def']
)
p99_rep_cust, med_rep_cust, mean_rep_cust, std_rep_cust = stats(
benchmark_samples['run_rep_cust']
)
p99_rep_unk, med_rep_unk, mean_rep_unk, std_rep_unk = stats(
benchmark_samples['run_rep_unk']
)
print(
f'\n[BENCHMARK] ParseJsonDefault: med {med_def:.2f}'
f' us/op | p99 {p99_def:.2f} us/op | mean {mean_def:.2f} ±'
f' {std_def:.2f} us/op'
)
print(
f'[BENCHMARK] ParseJsonCustom: med {med_cust:.2f} us/op'
f' | p99 {p99_cust:.2f} us/op | mean {mean_cust:.2f} ± {std_cust:.2f}'
' us/op'
)
print(
'[BENCHMARK] ParseJsonUnknownIgnored: '
f' med {med_unk:.2f} us/op | p99 {p99_unk:.2f} us/op | mean'
f' {mean_unk:.2f} ± {std_unk:.2f} us/op'
)
print(
'[BENCHMARK] ParseRepeatedJsonDefault: '
f' med {med_rep_def:.2f} us/item | p99 {p99_rep_def:.2f} us/item |'
f' mean {mean_rep_def:.2f} ± {std_rep_def:.2f} us/item'
)
print(
'[BENCHMARK] ParseRepeatedJsonCustom: '
f' med {med_rep_cust:.2f} us/item | p99 {p99_rep_cust:.2f} us/item |'
f' mean {mean_rep_cust:.2f} ± {std_rep_cust:.2f} us/item'
)
print(
'[BENCHMARK] ParseRepeatedJsonUnknownIgnored:'
f' med {med_rep_unk:.2f} us/item | p99 {p99_rep_unk:.2f} us/item |'
f' mean {mean_rep_unk:.2f} ± {std_rep_unk:.2f} us/item'
)
if __name__ == '__main__':
unittest.main()

133
python/google/protobuf/internal/json_format_test.py
View file

@ -1982,6 +1982,139 @@ class JsonFormatTest(JsonFormatBase):
{'armor': 1},
)
def testAliasedEnumValuesSerializeToSharedCustomString(self):
for enum_val in (
json_enumval_custom_string_pb2.Armor.ARMOR_SABATON,
json_enumval_custom_string_pb2.Armor.ARMOR_SOLLERET,
):
msg = json_enumval_custom_string_pb2.Knight(armor=enum_val)
json_output = json_format.MessageToJson(msg)
self.assertEqual(json.loads(json_output), {'armor': 'sabaton'})
def testNumericCustomStringOptionSerializesAsString(self):
msg = json_enumval_custom_string_pb2.Knight(
armor=json_enumval_custom_string_pb2.Armor.ARMOR_HACHI_MAI_DO
)
json_output = json_format.MessageToJson(msg)
self.assertEqual(json.loads(json_output), {'armor': '8'})
def testParseRawEnumNameWithCustomOptionOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse('{"armor": "ARMOR_GREAT_HELM"}', msg)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_GREAT_HELM
)
def testParseAliasedCustomStringOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse('{"armor": "sabaton"}', msg)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_SABATON
)
def testParseAliasedRawEnumNameOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse('{"armor": "ARMOR_SOLLERET"}', msg)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_SOLLERET
)
def testParseNumericCustomStringOptionOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse('{"armor": "8"}', msg)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_HACHI_MAI_DO
)
def testParseIntegerInputForEnumWithNumericCustomOptionOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse('{"armor": 8}', msg)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_HACHI_MAI_DO
)
def testParseCustomStringSingleElementArrayFails(self):
msg = json_enumval_custom_string_pb2.Knight()
with self.assertRaises(json_format.ParseError):
json_format.Parse('{"armor": ["gr8 helm"]}', msg)
def testParseBooleanInputForEnum(self):
# In Python, bool is a subclass of int (int(True) == 1). We document that
# JSON boolean literals like 'true' are accepted for enum fields and coerced
# to integer enum values.
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse('{"armor": true}', msg)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_GREAT_HELM
)
def testParseCustomStringCaseMismatchFails(self):
msg = json_enumval_custom_string_pb2.Knight()
with self.assertRaises(json_format.ParseError):
json_format.Parse('{"armor": "GR8 HELM"}', msg)
def testParseUnknownEnumStringFails(self):
msg = json_enumval_custom_string_pb2.Knight()
with self.assertRaises(json_format.ParseError):
json_format.Parse('{"armor": "UNKNOWN_ARMOR"}', msg)
def testParseUnknownEnumStringWithIgnoreUnknownFieldsOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse(
'{"armor": "UNKNOWN_ARMOR"}', msg, ignore_unknown_fields=True
)
self.assertEqual(
msg.armor, json_enumval_custom_string_pb2.Armor.ARMOR_UNKNOWN
)
def testParseMapFieldWithCustomEnumNameOk(self):
msg = json_enumval_custom_string_pb2.Knight()
json_format.Parse(
'{"armor_map": {"helmet": "gr8 helm", "boots": "sabaton", "chest": 8}}',
msg,
)
self.assertEqual(
msg.armor_map['helmet'],
json_enumval_custom_string_pb2.Armor.ARMOR_GREAT_HELM,
)
self.assertEqual(
msg.armor_map['boots'],
json_enumval_custom_string_pb2.Armor.ARMOR_SABATON,
)
self.assertEqual(
msg.armor_map['chest'],
json_enumval_custom_string_pb2.Armor.ARMOR_HACHI_MAI_DO,
)
def testGetCustomJsonEnumNames(self):
enum_type = json_enumval_custom_string_pb2.Armor.DESCRIPTOR
cache = {}
names = json_format._GetCustomJsonEnumNames(enum_type, cache)
self.assertIn('gr8 helm', names)
self.assertEqual(names['gr8 helm'].name, 'ARMOR_GREAT_HELM')
self.assertEqual(names['gr8 helm'].number, 1)
self.assertIn('sabaton', names)
self.assertEqual(names['sabaton'].number, 7)
self.assertIn(names['sabaton'].name, ('ARMOR_SABATON', 'ARMOR_SOLLERET'))
self.assertIn('8', names)
self.assertEqual(names['8'].name, 'ARMOR_HACHI_MAI_DO')
self.assertEqual(names['8'].number, 8)
self.assertNotIn('ARMOR_GORGET', names)
# Verify cache is populated and subsequent calls return the cached dict.
self.assertIn(enum_type, cache)
self.assertIs(cache[enum_type], names)
cached_names = json_format._GetCustomJsonEnumNames(enum_type, cache)
self.assertIs(cached_names, names)
def testGetCustomJsonEnumNamesNoCustomOptions(self):
enum_type = unittest_pb2.ForeignEnum.DESCRIPTOR
cache = {}
names = json_format._GetCustomJsonEnumNames(enum_type, cache)
self.assertEqual(names, {})
self.assertIn(enum_type, cache)
self.assertEqual(cache[enum_type], {})
if __name__ == '__main__':
unittest.main()

146
python/google/protobuf/json_format.py
View file

@ -194,6 +194,32 @@ class _Printer(object):
self.use_integers_for_enums = use_integers_for_enums
self.descriptor_pool = descriptor_pool
self.unquote_int64_if_possible = unquote_int64_if_possible
self._enumvalue_json_extension = None
def _GetEnumValueJsonExtension(self):
if self._enumvalue_json_extension is None:
# Options are always put on the default pool, so we only search the
# default pool.
try:
# Using reflection to FindExtensionByName is quite expensive, hence the
# gymnastics to cache it into the instance attribute
# _enumvalue_json_extension.
# TODO: b/551998570 - Over the longer term, we can consider putting this
# information in bootstrap files so that we don't have to rely on using
# reflection to perform this lookup at all.
self._enumvalue_json_extension = (
descriptor_pool.Default().FindExtensionByName('pb.enumvalue.json')
)
except KeyError:
self._enumvalue_json_extension = {}
return self._enumvalue_json_extension or None
def _GetJsonEnumValueOption(self, ev):
"""Helper to get the JsonEnumValueOptions for an enum value."""
extension_descriptor = self._GetEnumValueJsonExtension()
if extension_descriptor is None:
return None
return _GetJsonEnumValueOption(ev, extension_descriptor)
def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
js = self._MessageToJsonObject(message)
@ -289,7 +315,7 @@ class _Printer(object):
return None
enum_value = field.enum_type.values_by_number.get(value, None)
if enum_value is not None:
option = _GetJsonEnumValueOption(enum_value)
option = self._GetJsonEnumValueOption(enum_value)
if option is not None:
return option.string
return enum_value.name
@ -513,6 +539,20 @@ class _Parser(object):
self.descriptor_pool = descriptor_pool
self.max_recursion_depth = max_recursion_depth
self.recursion_depth = 0
self._custom_enum_names_cache = {}
self._enumvalue_json_extension = None
def _GetEnumValueJsonExtension(self):
if self._enumvalue_json_extension is None:
# Options are always put on the default pool, so we only search the
# default pool.
try:
self._enumvalue_json_extension = (
descriptor_pool.Default().FindExtensionByName('pb.enumvalue.json')
)
except KeyError:
self._enumvalue_json_extension = {}
return self._enumvalue_json_extension or None
def ConvertMessage(self, value, message, path):
"""Convert a JSON object into a message.
@ -848,7 +888,12 @@ class _Parser(object):
value_field = field.message_type.fields_by_name['value']
for key in value:
key_value = _ConvertScalarFieldValue(
key, key_field, '{0}.key'.format(path), True
key,
key_field,
'{0}.key'.format(path),
self._custom_enum_names_cache,
self._GetEnumValueJsonExtension(),
require_str=True,
)
if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
self.ConvertMessage(
@ -868,7 +913,13 @@ class _Parser(object):
def _ConvertAndSetScalar(self, message, field, js_value, path):
"""Convert scalar from js_value and assign it to message.field."""
try:
value = _ConvertScalarFieldValue(js_value, field, path)
value = _ConvertScalarFieldValue(
js_value,
field,
path,
self._custom_enum_names_cache,
self._GetEnumValueJsonExtension(),
)
if field.is_extension:
message.Extensions[field] = value
else:
@ -884,7 +935,13 @@ class _Parser(object):
repeated = message.Extensions[repeated_field]
else:
repeated = getattr(message, repeated_field.name)
value = _ConvertScalarFieldValue(js_value, repeated_field, path)
value = _ConvertScalarFieldValue(
js_value,
repeated_field,
path,
self._custom_enum_names_cache,
self._GetEnumValueJsonExtension(),
)
repeated.append(value)
except EnumStringValueParseError:
if not self.ignore_unknown_fields:
@ -900,39 +957,87 @@ class _Parser(object):
js_value,
map_field.message_type.fields_by_name['value'],
path,
self._custom_enum_names_cache,
self._GetEnumValueJsonExtension(),
)
)
except EnumStringValueParseError:
if not self.ignore_unknown_fields:
raise
def _GetJsonEnumValueOption(ev):
def _GetJsonEnumValueOption(ev, extension_descriptor):
"""Helper to get the JsonEnumValueOptions for an enum value.
Args:
ev: The EnumValueDescriptor.
extension_descriptor: The extension descriptor for 'pb.enumvalue.json'.
Returns:
The JsonEnumValueOptions message if the extension is present,
otherwise None.
"""
try:
extension_descriptor = descriptor_pool.Default().FindExtensionByName(
'pb.enumvalue.json'
)
except KeyError:
return None
if ev.GetOptions().HasExtension(extension_descriptor):
return ev.GetOptions().Extensions[extension_descriptor]
return None
def _ConvertScalarFieldValue(value, field, path, require_str=False):
def _GetCustomJsonEnumNames(
enum_type, custom_enum_names_cache, enumvalue_json_extension=None
):
"""Helper to get a mapping from custom JSON name to EnumValueDescriptor.
Args:
enum_type: The EnumDescriptor.
custom_enum_names_cache: A dict to store/lookup the cached map.
enumvalue_json_extension: The extension descriptor for 'pb.enumvalue.json',
or None to look it up in the default descriptor pool.
Returns:
A dict mapping custom JSON name strings to EnumValueDescriptors.
"""
if enum_type in custom_enum_names_cache:
return custom_enum_names_cache[enum_type]
custom_names = {}
if enumvalue_json_extension is None:
# Options are always put on the default pool, so we only search the default pool.
try:
enumvalue_json_extension = descriptor_pool.Default().FindExtensionByName(
'pb.enumvalue.json'
)
except KeyError:
enumvalue_json_extension = None
if enumvalue_json_extension is not None:
for ev in enum_type.values:
options = ev.GetOptions()
if options.HasExtension(enumvalue_json_extension):
option = options.Extensions[enumvalue_json_extension]
if option.HasField('string'):
custom_names[option.string] = ev
custom_enum_names_cache[enum_type] = custom_names
return custom_names
def _ConvertScalarFieldValue(
value,
field,
path,
custom_enum_names_cache,
enumvalue_json_extension=None,
require_str=False,
):
"""Convert a single scalar field value.
Args:
value: A scalar value to convert the scalar field value.
field: The descriptor of the field to convert.
path: parent path to log parse error info.
custom_enum_names_cache: A dict to store/lookup custom enum names.
enumvalue_json_extension: The extension descriptor for 'pb.enumvalue.json',
or None if not loaded.
require_str: If True, the field value must be a str.
Returns:
@ -968,12 +1073,17 @@ def _ConvertScalarFieldValue(value, field, path, require_str=False):
# Convert an enum value.
enum_value = field.enum_type.values_by_name.get(value, None)
# First check to see if we have a custom enum string.
if enum_value is None:
for ev in field.enum_type.values:
option = _GetJsonEnumValueOption(ev)
if option is not None and option.string == value:
enum_value = ev
break
if (
enum_value is None
and isinstance(value, str)
and enumvalue_json_extension is not None
):
custom_names = _GetCustomJsonEnumNames(
field.enum_type,
custom_enum_names_cache,
enumvalue_json_extension,
)
enum_value = custom_names.get(value, None)
# If not, try parsing it as an integer.
if enum_value is None:
try: