Internal change.

PiperOrigin-RevId: 927241088

python/google/protobuf/internal/containers.py

@@ -451,11 +451,11 @@ class ScalarMap(MutableMapping[_K, _V]):
     return self
 
   def __getitem__(self, key: _K) -> _V:
+    key = self._key_checker.CheckValue(key)
     try:
       return self._values[key]
     except KeyError:
       self._AssureWritable()
-      key = self._key_checker.CheckValue(key)
       val = self._value_checker.DefaultValue()
       self._values[key] = val
       return val
@@ -463,7 +463,7 @@ class ScalarMap(MutableMapping[_K, _V]):
   def __contains__(self, item: _K) -> bool:
     # We check the key's type to match the strong-typing flavor of the API.
     # Also this makes it easier to match the behavior of the C++ implementation.
-    self._key_checker.CheckValue(item)
+    item = self._key_checker.CheckValue(item)
     return item in self._values
 
   @overload

python/google/protobuf/internal/message_benchmark.py

@@ -96,7 +96,7 @@ def bench_assign_bytes_with_conversion(state: google_benchmark.State):
     state.pause_timing()
     msg.Clear()
     state.resume_timing()
-    msg.optional_bytes = arr.tobytes()
+    msg.optional_bytes = memoryview(arr)
 
 
 if __name__ == '__main__':

python/google/protobuf/pyext/map_container.cc

@@ -14,8 +14,10 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
 
+#include "absl/strings/string_view.h"
 #include "google/protobuf/map.h"
 #include "google/protobuf/map_field.h"
 #include "google/protobuf/message.h"
@@ -84,26 +86,7 @@ Message* MapContainer::GetMutableMessage() {
   return cmessage::AssureWritable(parent);
 }
 
-// Consumes a reference on the Python string object.
-static bool PyStringToSTL(PyObject* py_string, std::string* stl_string) {
-  char* value;
-  Py_ssize_t value_len;
-
-  if (!py_string) {
-    return false;
-  }
-  if (PyBytes_AsStringAndSize(py_string, &value, &value_len) < 0) {
-    Py_DECREF(py_string);
-    return false;
-  } else {
-    stl_string->assign(value, value_len);
-    Py_DECREF(py_string);
-    return true;
-  }
-}
-
-static bool PythonToMapKey(MapContainer* self, PyObject* obj, MapKey* key,
-                           std::string* key_string) {
+static bool PythonToMapKey(MapContainer* self, PyObject* obj, MapKey* key) {
   const FieldDescriptor* field_descriptor =
       self->parent_field_descriptor->message_type()->map_key();
   switch (field_descriptor->cpp_type()) {
@@ -133,10 +116,12 @@ static bool PythonToMapKey(MapContainer* self, PyObject* obj, MapKey* key,
       break;
     }
     case FieldDescriptor::CPPTYPE_STRING: {
-      if (!PyStringToSTL(CheckString(obj, field_descriptor), key_string)) {
+      std::optional<absl::string_view> key_view =
+          CheckString(obj, field_descriptor);
+      if (!key_view.has_value()) {
         return false;
       }
-      key->SetStringValue(*key_string);
+      key->SetStringValue(*key_view);
       break;
     }
     default:
@@ -246,12 +231,12 @@ static bool PythonToMapValueRef(MapContainer* self, PyObject* obj,
       return true;
     }
     case FieldDescriptor::CPPTYPE_STRING: {
-      std::string str;
-      if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) {
-        return false;
+      std::optional<absl::string_view> value =
+          CheckString(obj, field_descriptor);
+      if (value.has_value()) {
+        value_ref->SetStringValue(*value);
       }
-      value_ref->SetStringValue(str);
-      return true;
+      return value.has_value();
     }
     case FieldDescriptor::CPPTYPE_ENUM: {
       PROTOBUF_CHECK_GET_INT32(obj, value, false);
@@ -340,10 +325,9 @@ int MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
 
   const Message* message = self->parent->message;
   const Reflection* reflection = message->GetReflection();
-  std::string map_key_string;
   MapKey map_key;
 
-  if (!PythonToMapKey(self, key, &map_key, &map_key_string)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return -1;
   }
 
@@ -386,11 +370,10 @@ PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self,
   Message* message = self->GetMutableMessage();
   if (message == nullptr) return nullptr;
   const Reflection* reflection = message->GetReflection();
-  std::string map_key_string;
   MapKey map_key;
   MapValueRef value;
 
-  if (!PythonToMapKey(self, key, &map_key, &map_key_string)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return nullptr;
   }
 
@@ -409,11 +392,10 @@ int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key,
   Message* message = self->GetMutableMessage();
   if (message == nullptr) return -1;
   const Reflection* reflection = message->GetReflection();
-  std::string map_key_string;
   MapKey map_key;
   MapValueRef value;
 
-  if (!PythonToMapKey(self, key, &map_key, &map_key_string)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return -1;
   }
 
@@ -635,12 +617,11 @@ int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key,
   Message* message = self->GetMutableMessage();
   if (message == nullptr) return -1;
   const Reflection* reflection = message->GetReflection();
-  std::string map_key_string;
   MapKey map_key;
 
   self->version++;
 
-  if (!PythonToMapKey(self, key, &map_key, &map_key_string)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return -1;
   }
 
@@ -679,11 +660,10 @@ PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self,
   Message* message = self->GetMutableMessage();
   if (message == nullptr) return nullptr;
   const Reflection* reflection = message->GetReflection();
-  std::string map_key_string;
   MapKey map_key;
   MapValueRef value;
 
-  if (!PythonToMapKey(self, key, &map_key, &map_key_string)) {
+  if (!PythonToMapKey(self, key, &map_key)) {
     return nullptr;
   }
 

python/google/protobuf/pyext/message.cc

@@ -19,6 +19,7 @@
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <optional>
 #include <set>
 #include <string>
 #include <utility>
@@ -605,87 +606,48 @@ bool CheckAndGetBool(PyObject* arg, bool* value) {
   return true;
 }
 
-// Checks whether the given object (which must be "bytes" or "unicode") contains
-// valid UTF-8.
-bool IsValidUTF8(PyObject* obj) {
-  if (PyBytes_Check(obj)) {
-    PyObject* unicode = PyUnicode_FromEncodedObject(obj, "utf-8", nullptr);
-
-    // Clear the error indicator; we report our own error when desired.
-    PyErr_Clear();
-
-    if (unicode) {
-      Py_DECREF(unicode);
-      return true;
-    } else {
-      return false;
-    }
-  } else {
-    // Unicode object, known to be valid UTF-8.
-    return true;
-  }
-}
-
 bool AllowInvalidUTF8(const FieldDescriptor* field) { return false; }
 
-PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor) {
+std::optional<absl::string_view> CheckString(
+    PyObject* arg, const FieldDescriptor* descriptor) {
   ABSL_DCHECK(descriptor->type() == FieldDescriptor::TYPE_STRING ||
               descriptor->type() == FieldDescriptor::TYPE_BYTES);
   if (descriptor->type() == FieldDescriptor::TYPE_STRING) {
-    if (!PyBytes_Check(arg) && !PyUnicode_Check(arg)) {
-      FormatTypeError(arg, "bytes, unicode");
-      return nullptr;
+    if (PyUnicode_Check(arg)) {
+      // Use the str object's cached UTF-8 representation — no allocation.
+      // The pointer is valid as long as arg is alive.
+      Py_ssize_t utf8_len;
+      const char* utf8 = PyUnicode_AsUTF8AndSize(arg, &utf8_len);
+      if (utf8 == nullptr) return std::nullopt;
+      return absl::string_view(utf8, utf8_len);
     }
-
-    if (!IsValidUTF8(arg) && !AllowInvalidUTF8(descriptor)) {
-      PyObject* repr = PyObject_Repr(arg);
-      PyErr_Format(PyExc_ValueError,
-                   "%s has type str, but isn't valid UTF-8 "
-                   "encoding. Non-UTF-8 strings must be converted to "
-                   "unicode objects before being added.",
-                   PyString_AsString(repr));
-      Py_DECREF(repr);
-      return nullptr;
-    }
-  } else if (!PyBytes_Check(arg)) {
-    FormatTypeError(arg, "bytes");
-    return nullptr;
   }
 
-  PyObject* encoded_string = nullptr;
-  if (descriptor->type() == FieldDescriptor::TYPE_STRING) {
-    if (PyBytes_Check(arg)) {
-      // The bytes were already validated as correctly encoded UTF-8 above.
-      encoded_string = arg;  // Already encoded.
-      Py_INCREF(encoded_string);
-    } else {
-      encoded_string = PyUnicode_AsEncodedString(arg, "utf-8", nullptr);
+  if (PyBytes_Check(arg)) {
+    absl::string_view value(PyBytes_AS_STRING(arg), PyBytes_GET_SIZE(arg));
+    if (descriptor->type() == FieldDescriptor::TYPE_STRING &&
+        !AllowInvalidUTF8(descriptor)) {
+      PyObject* unicode =
+          PyUnicode_FromStringAndSize(value.data(), value.size());
+      if (unicode == nullptr) {
+        return std::nullopt;
+      }
+      Py_DECREF(unicode);
     }
-  } else {
-    // In this case field type is "bytes".
-    encoded_string = arg;
-    Py_INCREF(encoded_string);
+    return value;
   }
-
-  return encoded_string;
+  FormatTypeError(arg, "bytes, unicode");
+  return std::nullopt;
 }
 
 bool CheckAndSetString(PyObject* arg, Message* message,
                        const FieldDescriptor* descriptor,
                        const Reflection* reflection, bool append, int index) {
-  ScopedPyObjectPtr encoded_string(CheckString(arg, descriptor));
-
-  if (encoded_string.get() == nullptr) {
+  std::optional<absl::string_view> value = CheckString(arg, descriptor);
+  if (!value.has_value()) {
     return false;
   }
-
-  char* value;
-  Py_ssize_t value_len;
-  if (PyBytes_AsStringAndSize(encoded_string.get(), &value, &value_len) < 0) {
-    return false;
-  }
-
-  std::string value_string(value, value_len);
+  std::string value_string(*value);
   if (append) {
     reflection->AddString(message, descriptor, std::move(value_string));
   } else if (index < 0) {
@@ -2586,22 +2548,16 @@ PyObject* Contains(CMessage* self, PyObject* arg) {
       const Reflection* reflection = message->GetReflection();
       const FieldDescriptor* map_field = descriptor->FindFieldByName("fields");
       const FieldDescriptor* key_field = map_field->message_type()->map_key();
-      ScopedPyObjectPtr py_string(CheckString(arg, key_field));
-      if (py_string.get() == nullptr) {
+      MapKey map_key;
+      std::optional<absl::string_view> key_string = CheckString(arg, key_field);
+      if (!key_string.has_value()) {
+        PyErr_Clear();
         PyErr_SetString(PyExc_TypeError,
                         "The key passed to Struct message must be a str.");
         return nullptr;
       }
-      char* value;
-      Py_ssize_t value_len;
-      if (PyBytes_AsStringAndSize(py_string.get(), &value, &value_len) < 0) {
-        Py_RETURN_FALSE;
-      }
-      std::string key_str;
-      key_str.assign(value, value_len);
+      map_key.SetStringValue(*key_string);
 
-      MapKey map_key;
-      map_key.SetStringValue(key_str);
       return PyBool_FromLong(MessageReflectionFriend::ContainsMapKey(
           reflection, *message, map_field, map_key));
     }

python/google/protobuf/pyext/message.h

@@ -15,6 +15,9 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 
+#include <cstdint>
+#include <optional>
+
 #include "absl/strings/string_view.h"
 #include "google/protobuf/pyext/lazy_unique_ptr.h"
 #include "google/protobuf/pyext/weak_value_map.h"
@@ -332,7 +335,11 @@ bool CheckAndGetInteger(PyObject* arg, T* value);
 bool CheckAndGetDouble(PyObject* arg, double* value);
 bool CheckAndGetFloat(PyObject* arg, float* value);
 bool CheckAndGetBool(PyObject* arg, bool* value);
-PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor);
+
+// Validates arg for a string or bytes field, and returns the string view if
+// valid. Returns std::nullopt and sets a Python exception on failure.
+std::optional<absl::string_view> CheckString(PyObject* arg,
+                                             const FieldDescriptor* descriptor);
 bool CheckAndSetString(PyObject* arg, Message* message,
                        const FieldDescriptor* descriptor,
                        const Reflection* reflection, bool append, int index);